2023-09-06 20:13:30,455 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@10aa41f2 2023-09-06 20:13:30,472 main DEBUG Took 0.014590 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-09-06 20:13:30,473 main DEBUG PluginManager 'Core' found 129 plugins 2023-09-06 20:13:30,473 main DEBUG PluginManager 'Level' found 0 plugins 2023-09-06 20:13:30,475 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-09-06 20:13:30,476 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:13:30,485 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-09-06 20:13:30,501 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:13:30,503 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:13:30,504 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:13:30,504 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:13:30,504 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:13:30,504 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:13:30,505 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:13:30,505 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:13:30,506 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:13:30,506 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:13:30,507 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:13:30,507 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:13:30,508 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:13:30,508 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:13:30,509 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:13:30,509 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:13:30,510 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:13:30,510 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:13:30,511 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:13:30,511 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:13:30,512 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:13:30,512 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:13:30,513 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:13:30,513 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:13:30,513 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:13:30,514 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-09-06 20:13:30,516 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:13:30,519 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-09-06 20:13:30,520 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-09-06 20:13:30,521 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-09-06 20:13:30,522 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-09-06 20:13:30,522 main DEBUG PluginManager 'Converter' found 47 plugins 2023-09-06 20:13:30,535 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-09-06 20:13:30,538 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-09-06 20:13:30,539 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-09-06 20:13:30,539 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-09-06 20:13:30,540 main DEBUG createAppenders(={Console}) 2023-09-06 20:13:30,540 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@10aa41f2 initialized 2023-09-06 20:13:30,541 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@10aa41f2 2023-09-06 20:13:30,541 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@10aa41f2 OK. 2023-09-06 20:13:30,541 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-09-06 20:13:30,542 main DEBUG OutputStream closed 2023-09-06 20:13:30,542 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-09-06 20:13:30,542 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-09-06 20:13:30,543 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3023df74 OK 2023-09-06 20:13:30,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-09-06 20:13:30,600 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-09-06 20:13:30,601 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-09-06 20:13:30,603 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-09-06 20:13:30,604 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-09-06 20:13:30,604 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-09-06 20:13:30,604 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-09-06 20:13:30,605 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-09-06 20:13:30,605 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-09-06 20:13:30,606 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-09-06 20:13:30,606 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-09-06 20:13:30,607 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-09-06 20:13:30,607 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-09-06 20:13:30,608 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-09-06 20:13:30,608 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-09-06 20:13:30,608 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-09-06 20:13:30,609 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-09-06 20:13:30,610 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-09-06 20:13:30,617 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-06 20:13:30,618 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@54a7079e) with optional ClassLoader: null 2023-09-06 20:13:30,618 main DEBUG Shutdown hook enabled. Registering a new one. 2023-09-06 20:13:30,619 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@54a7079e] started OK. 2023-09-06T20:13:30,936 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e 2023-09-06 20:13:30,941 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-09-06 20:13:30,941 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-06T20:13:30,954 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithAdaptivePolicy timeout: 13 mins 2023-09-06T20:13:30,967 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-09-06T20:13:30,971 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/cluster_e2672345-5de3-05ed-1693-e29dc2a9b7f5, deleteOnExit=true 2023-09-06T20:13:30,972 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-09-06T20:13:30,972 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/test.cache.data in system properties and HBase conf 2023-09-06T20:13:30,973 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/hadoop.tmp.dir in system properties and HBase conf 2023-09-06T20:13:30,973 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/hadoop.log.dir in system properties and HBase conf 2023-09-06T20:13:30,973 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-06T20:13:30,974 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-06T20:13:30,974 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-06T20:13:31,124 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-09-06T20:13:31,604 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-09-06T20:13:31,608 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:13:31,609 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:13:31,609 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-06T20:13:31,610 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:13:31,610 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-06T20:13:31,611 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-06T20:13:31,611 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:13:31,612 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:13:31,612 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-06T20:13:31,613 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/nfs.dump.dir in system properties and HBase conf 2023-09-06T20:13:31,613 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/java.io.tmpdir in system properties and HBase conf 2023-09-06T20:13:31,613 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:13:31,614 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-06T20:13:31,614 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-09-06T20:13:32,145 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:13:32,151 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:13:32,507 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-09-06T20:13:32,661 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@66a64b4b via org.mortbay.log.Slf4jLog 2023-09-06T20:13:32,683 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:13:32,736 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:13:32,780 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/java.io.tmpdir/Jetty_localhost_42751_hdfs____7qvszj/webapp 2023-09-06T20:13:32,953 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42751 2023-09-06T20:13:32,978 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:13:32,979 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:13:33,627 WARN [Listener at localhost/42081 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:13:33,725 WARN [Listener at localhost/42081 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-09-06T20:13:33,745 WARN [Listener at localhost/42081 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:13:33,752 INFO [Listener at localhost/42081 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:13:33,759 INFO [Listener at localhost/42081 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/java.io.tmpdir/Jetty_localhost_44915_datanode____t2k9vo/webapp 2023-09-06T20:13:33,883 INFO [Listener at localhost/42081 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44915 2023-09-06T20:13:34,135 WARN [Listener at localhost/35961 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:13:34,509 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-09-06T20:13:34,721 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x7645e8314616a56a: Processing first storage report for DS-38d94f8a-49ee-4b1d-9e2d-02a906c5d972 from datanode bfc4c0f9-6833-40e2-8540-c91c5ba0f41a 2023-09-06T20:13:34,723 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x7645e8314616a56a: from storage DS-38d94f8a-49ee-4b1d-9e2d-02a906c5d972 node DatanodeRegistration(127.0.0.1:42057, datanodeUuid=bfc4c0f9-6833-40e2-8540-c91c5ba0f41a, infoPort=44009, infoSecurePort=0, ipcPort=35961, storageInfo=lv=-57;cid=testClusterID;nsid=1065738550;c=1694031212233), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-09-06T20:13:34,723 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x7645e8314616a56a: Processing first storage report for DS-8c05a1d7-0fc7-4602-b547-5f7407710353 from datanode bfc4c0f9-6833-40e2-8540-c91c5ba0f41a 2023-09-06T20:13:34,723 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x7645e8314616a56a: from storage DS-8c05a1d7-0fc7-4602-b547-5f7407710353 node DatanodeRegistration(127.0.0.1:42057, datanodeUuid=bfc4c0f9-6833-40e2-8540-c91c5ba0f41a, infoPort=44009, infoSecurePort=0, ipcPort=35961, storageInfo=lv=-57;cid=testClusterID;nsid=1065738550;c=1694031212233), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:13:34,729 DEBUG [Listener at localhost/35961 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e 2023-09-06T20:13:34,805 INFO [Listener at localhost/35961 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/cluster_e2672345-5de3-05ed-1693-e29dc2a9b7f5/zookeeper_0, clientPort=58481, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/cluster_e2672345-5de3-05ed-1693-e29dc2a9b7f5/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/cluster_e2672345-5de3-05ed-1693-e29dc2a9b7f5/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-09-06T20:13:34,822 INFO [Listener at localhost/35961 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=58481 2023-09-06T20:13:34,830 INFO [Listener at localhost/35961 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:13:34,832 INFO [Listener at localhost/35961 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:13:35,542 INFO [Listener at localhost/35961 {}] util.FSUtils(461): Created version file at hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236 with version=8 2023-09-06T20:13:35,543 INFO [Listener at localhost/35961 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/hbase-staging 2023-09-06T20:13:35,778 DEBUG [Listener at localhost/35961 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-09-06T20:13:36,544 INFO [Listener at localhost/35961 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-09-06T20:13:36,586 INFO [Listener at localhost/35961 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:13:36,587 INFO [Listener at localhost/35961 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-06T20:13:36,587 INFO [Listener at localhost/35961 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-06T20:13:36,587 INFO [Listener at localhost/35961 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:13:36,587 INFO [Listener at localhost/35961 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-06T20:13:36,767 INFO [Listener at localhost/35961 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-06T20:13:36,894 INFO [Listener at localhost/35961 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-09-06T20:13:36,913 DEBUG [Listener at localhost/35961 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-09-06T20:13:36,921 INFO [Listener at localhost/35961 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-06T20:13:36,981 DEBUG [Listener at localhost/35961 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 2750 (auto-detected) 2023-09-06T20:13:36,983 DEBUG [Listener at localhost/35961 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-09-06T20:13:37,029 INFO [Listener at localhost/35961 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:35153 2023-09-06T20:13:37,045 INFO [Listener at localhost/35961 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:13:37,048 INFO [Listener at localhost/35961 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:13:37,071 INFO [Listener at localhost/35961 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:35153 connecting to ZooKeeper ensemble=127.0.0.1:58481 2023-09-06T20:13:37,124 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:351530x0, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-06T20:13:37,128 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:35153-0x100eeea591f0000 connected 2023-09-06T20:13:37,162 DEBUG [Listener at localhost/35961 {}] zookeeper.ZKUtil(113): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2023-09-06T20:13:37,166 DEBUG [Listener at localhost/35961 {}] zookeeper.ZKUtil(113): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-09-06T20:13:37,170 DEBUG [Listener at localhost/35961 {}] zookeeper.ZKUtil(113): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2023-09-06T20:13:37,179 DEBUG [Listener at localhost/35961 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35153 2023-09-06T20:13:37,179 DEBUG [Listener at localhost/35961 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35153 2023-09-06T20:13:37,182 DEBUG [Listener at localhost/35961 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35153 2023-09-06T20:13:37,182 DEBUG [Listener at localhost/35961 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35153 2023-09-06T20:13:37,183 DEBUG [Listener at localhost/35961 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35153 2023-09-06T20:13:37,196 INFO [Listener at localhost/35961 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236, hbase.cluster.distributed=false 2023-09-06T20:13:37,269 INFO [Listener at localhost/35961 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-09-06T20:13:37,270 INFO [Listener at localhost/35961 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:13:37,270 INFO [Listener at localhost/35961 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-06T20:13:37,271 INFO [Listener at localhost/35961 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-06T20:13:37,271 INFO [Listener at localhost/35961 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:13:37,271 INFO [Listener at localhost/35961 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-06T20:13:37,276 INFO [Listener at localhost/35961 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-06T20:13:37,278 INFO [Listener at localhost/35961 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-06T20:13:37,279 INFO [Listener at localhost/35961 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:33275 2023-09-06T20:13:37,281 INFO [Listener at localhost/35961 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-09-06T20:13:37,290 DEBUG [Listener at localhost/35961 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-09-06T20:13:37,291 INFO [Listener at localhost/35961 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:13:37,295 INFO [Listener at localhost/35961 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:13:37,299 INFO [Listener at localhost/35961 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:33275 connecting to ZooKeeper ensemble=127.0.0.1:58481 2023-09-06T20:13:37,305 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:332750x0, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-06T20:13:37,306 DEBUG [Listener at localhost/35961 {}] zookeeper.ZKUtil(113): regionserver:332750x0, quorum=127.0.0.1:58481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2023-09-06T20:13:37,306 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:33275-0x100eeea591f0001 connected 2023-09-06T20:13:37,308 DEBUG [Listener at localhost/35961 {}] zookeeper.ZKUtil(113): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-09-06T20:13:37,310 DEBUG [Listener at localhost/35961 {}] zookeeper.ZKUtil(113): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2023-09-06T20:13:37,314 DEBUG [Listener at localhost/35961 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33275 2023-09-06T20:13:37,314 DEBUG [Listener at localhost/35961 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33275 2023-09-06T20:13:37,315 DEBUG [Listener at localhost/35961 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33275 2023-09-06T20:13:37,316 DEBUG [Listener at localhost/35961 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33275 2023-09-06T20:13:37,318 DEBUG [Listener at localhost/35961 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33275 2023-09-06T20:13:37,322 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /hbase/backup-masters/jenkins-hbase3.apache.org,35153,1694031215758 2023-09-06T20:13:37,332 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-09-06T20:13:37,333 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-09-06T20:13:37,334 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/jenkins-hbase3.apache.org,35153,1694031215758 2023-09-06T20:13:37,352 DEBUG [M:0;jenkins-hbase3:35153 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:35153 2023-09-06T20:13:37,357 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2023-09-06T20:13:37,357 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2023-09-06T20:13:37,358 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:13:37,358 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:13:37,359 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2023-09-06T20:13:37,361 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/jenkins-hbase3.apache.org,35153,1694031215758 from backup master directory 2023-09-06T20:13:37,362 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2023-09-06T20:13:37,365 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/jenkins-hbase3.apache.org,35153,1694031215758 2023-09-06T20:13:37,365 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-09-06T20:13:37,365 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-09-06T20:13:37,366 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-06T20:13:37,367 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,35153,1694031215758 2023-09-06T20:13:37,370 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-09-06T20:13:37,372 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-09-06T20:13:37,877 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/hbase.id with ID: 9b3154a5-e59d-461f-ad9d-9e9626a73818 2023-09-06T20:13:37,928 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:13:37,945 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:13:37,945 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:13:38,398 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:13:38,401 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-09-06T20:13:38,421 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-09-06T20:13:38,422 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-09-06T20:13:38,423 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:38,432 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:38,434 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:13:38,510 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store 2023-09-06T20:13:38,941 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-09-06T20:13:38,941 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:13:38,942 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-06T20:13:38,942 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:13:38,942 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:13:38,942 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-06T20:13:38,942 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:13:38,942 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:13:38,942 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:13:38,945 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/.initializing 2023-09-06T20:13:38,945 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/WALs/jenkins-hbase3.apache.org,35153,1694031215758 2023-09-06T20:13:38,957 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-06T20:13:38,976 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C35153%2C1694031215758, suffix=, logDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/WALs/jenkins-hbase3.apache.org,35153,1694031215758, archiveDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/oldWALs, maxLogs=10 2023-09-06T20:13:39,010 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/WALs/jenkins-hbase3.apache.org,35153,1694031215758/jenkins-hbase3.apache.org%2C35153%2C1694031215758.1694031218985, exclude list is [], retry=0 2023-09-06T20:13:39,047 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42057,DS-38d94f8a-49ee-4b1d-9e2d-02a906c5d972,DISK] 2023-09-06T20:13:39,058 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:39,143 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/WALs/jenkins-hbase3.apache.org,35153,1694031215758/jenkins-hbase3.apache.org%2C35153%2C1694031215758.1694031218985 2023-09-06T20:13:39,144 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:42057,DS-38d94f8a-49ee-4b1d-9e2d-02a906c5d972,DISK]] 2023-09-06T20:13:39,144 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:13:39,145 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:13:39,150 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:13:39,151 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:13:39,220 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:13:39,253 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-09-06T20:13:39,258 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:39,262 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:13:39,262 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:13:39,265 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-09-06T20:13:39,266 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:39,268 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:13:39,268 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:13:39,271 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-09-06T20:13:39,271 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:39,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:13:39,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:13:39,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-09-06T20:13:39,277 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:39,278 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:13:39,284 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:13:39,286 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:13:39,300 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-09-06T20:13:39,314 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:13:39,319 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:13:39,321 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69274460, jitterRate=0.03226989507675171}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-09-06T20:13:39,330 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:13:39,338 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-09-06T20:13:39,390 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25861239, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:13:39,447 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-09-06T20:13:39,470 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-09-06T20:13:39,471 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-09-06T20:13:39,474 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-09-06T20:13:39,476 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-09-06T20:13:39,483 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 7 msec 2023-09-06T20:13:39,484 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-09-06T20:13:39,524 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-09-06T20:13:39,545 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2023-09-06T20:13:39,548 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/balancer already deleted, retry=false 2023-09-06T20:13:39,552 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-09-06T20:13:39,553 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2023-09-06T20:13:39,555 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/normalizer already deleted, retry=false 2023-09-06T20:13:39,558 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-09-06T20:13:39,563 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2023-09-06T20:13:39,565 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/switch/split already deleted, retry=false 2023-09-06T20:13:39,566 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2023-09-06T20:13:39,570 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/switch/merge already deleted, retry=false 2023-09-06T20:13:39,587 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2023-09-06T20:13:39,588 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/snapshot-cleanup already deleted, retry=false 2023-09-06T20:13:39,593 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2023-09-06T20:13:39,593 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2023-09-06T20:13:39,594 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:13:39,594 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:13:39,594 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,35153,1694031215758, sessionid=0x100eeea591f0000, setting cluster-up flag (Was=false) 2023-09-06T20:13:39,611 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:13:39,611 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:13:39,619 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2023-09-06T20:13:39,621 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,35153,1694031215758 2023-09-06T20:13:39,627 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:13:39,627 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:13:39,634 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2023-09-06T20:13:39,638 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,35153,1694031215758 2023-09-06T20:13:39,736 DEBUG [RS:0;jenkins-hbase3:33275 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:33275 2023-09-06T20:13:39,738 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(997): ClusterId : 9b3154a5-e59d-461f-ad9d-9e9626a73818 2023-09-06T20:13:39,742 DEBUG [RS:0;jenkins-hbase3:33275 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-09-06T20:13:39,749 DEBUG [RS:0;jenkins-hbase3:33275 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-09-06T20:13:39,749 DEBUG [RS:0;jenkins-hbase3:33275 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-09-06T20:13:39,756 DEBUG [RS:0;jenkins-hbase3:33275 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-09-06T20:13:39,757 DEBUG [RS:0;jenkins-hbase3:33275 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@178c2812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:13:39,762 DEBUG [RS:0;jenkins-hbase3:33275 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@984dffd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-06T20:13:39,766 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-09-06T20:13:39,766 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-09-06T20:13:39,766 DEBUG [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-09-06T20:13:39,769 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,35153,1694031215758 with isa=jenkins-hbase3.apache.org/172.31.12.81:33275, startcode=1694031217268 2023-09-06T20:13:39,790 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-09-06T20:13:39,792 DEBUG [RS:0;jenkins-hbase3:33275 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-06T20:13:39,799 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-09-06T20:13:39,804 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-09-06T20:13:39,815 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,35153,1694031215758 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-09-06T20:13:39,825 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:13:39,825 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:13:39,825 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:13:39,826 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:13:39,826 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-09-06T20:13:39,826 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:13:39,826 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-06T20:13:39,826 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:13:39,848 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1694031249848 2023-09-06T20:13:39,851 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-09-06T20:13:39,854 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-09-06T20:13:39,855 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-09-06T20:13:39,861 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:39,861 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-06T20:13:39,869 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-09-06T20:13:39,869 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38187, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-06T20:13:39,881 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35153 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3181) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:590) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:39,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-09-06T20:13:39,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-09-06T20:13:39,883 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-09-06T20:13:39,883 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-09-06T20:13:39,884 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:39,887 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-09-06T20:13:39,889 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-09-06T20:13:39,889 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-09-06T20:13:39,906 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-09-06T20:13:39,907 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-09-06T20:13:39,909 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694031219909,5,FailOnTimeoutGroup] 2023-09-06T20:13:39,909 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694031219909,5,FailOnTimeoutGroup] 2023-09-06T20:13:39,909 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:39,909 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-09-06T20:13:39,911 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:39,912 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:39,934 DEBUG [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-09-06T20:13:39,934 WARN [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-09-06T20:13:40,035 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,35153,1694031215758 with isa=jenkins-hbase3.apache.org/172.31.12.81:33275, startcode=1694031217268 2023-09-06T20:13:40,042 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35153 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:40,055 DEBUG [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236 2023-09-06T20:13:40,055 DEBUG [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:42081 2023-09-06T20:13:40,055 DEBUG [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=-1 2023-09-06T20:13:40,061 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2023-09-06T20:13:40,062 DEBUG [RS:0;jenkins-hbase3:33275 {}] zookeeper.ZKUtil(111): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:40,062 WARN [RS:0;jenkins-hbase3:33275 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-06T20:13:40,063 INFO [RS:0;jenkins-hbase3:33275 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:13:40,063 DEBUG [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/WALs/jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:40,066 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,33275,1694031217268] 2023-09-06T20:13:40,086 DEBUG [RS:0;jenkins-hbase3:33275 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-09-06T20:13:40,103 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-09-06T20:13:40,123 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-09-06T20:13:40,128 INFO [RS:0;jenkins-hbase3:33275 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-09-06T20:13:40,128 INFO [RS:0;jenkins-hbase3:33275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:40,132 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-09-06T20:13:40,141 INFO [RS:0;jenkins-hbase3:33275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:40,141 DEBUG [RS:0;jenkins-hbase3:33275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:13:40,142 DEBUG [RS:0;jenkins-hbase3:33275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:13:40,142 DEBUG [RS:0;jenkins-hbase3:33275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:13:40,142 DEBUG [RS:0;jenkins-hbase3:33275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:13:40,142 DEBUG [RS:0;jenkins-hbase3:33275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:13:40,143 DEBUG [RS:0;jenkins-hbase3:33275 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-06T20:13:40,143 DEBUG [RS:0;jenkins-hbase3:33275 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:13:40,143 DEBUG [RS:0;jenkins-hbase3:33275 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:13:40,144 DEBUG [RS:0;jenkins-hbase3:33275 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:13:40,144 DEBUG [RS:0;jenkins-hbase3:33275 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:13:40,144 DEBUG [RS:0;jenkins-hbase3:33275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:13:40,144 DEBUG [RS:0;jenkins-hbase3:33275 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-06T20:13:40,145 DEBUG [RS:0;jenkins-hbase3:33275 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-06T20:13:40,150 INFO [RS:0;jenkins-hbase3:33275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:40,150 INFO [RS:0;jenkins-hbase3:33275 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:40,150 INFO [RS:0;jenkins-hbase3:33275 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:40,150 INFO [RS:0;jenkins-hbase3:33275 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:40,151 INFO [RS:0;jenkins-hbase3:33275 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33275,1694031217268-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-06T20:13:40,172 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-09-06T20:13:40,176 INFO [RS:0;jenkins-hbase3:33275 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33275,1694031217268-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:40,199 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,33275,1694031217268 started 2023-09-06T20:13:40,199 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,33275,1694031217268, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:33275, sessionid=0x100eeea591f0001 2023-09-06T20:13:40,200 DEBUG [RS:0;jenkins-hbase3:33275 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-09-06T20:13:40,200 DEBUG [RS:0;jenkins-hbase3:33275 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:40,200 DEBUG [RS:0;jenkins-hbase3:33275 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,33275,1694031217268' 2023-09-06T20:13:40,200 DEBUG [RS:0;jenkins-hbase3:33275 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2023-09-06T20:13:40,201 DEBUG [RS:0;jenkins-hbase3:33275 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2023-09-06T20:13:40,202 DEBUG [RS:0;jenkins-hbase3:33275 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-09-06T20:13:40,202 DEBUG [RS:0;jenkins-hbase3:33275 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-09-06T20:13:40,202 DEBUG [RS:0;jenkins-hbase3:33275 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:40,202 DEBUG [RS:0;jenkins-hbase3:33275 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,33275,1694031217268' 2023-09-06T20:13:40,202 DEBUG [RS:0;jenkins-hbase3:33275 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2023-09-06T20:13:40,203 DEBUG [RS:0;jenkins-hbase3:33275 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2023-09-06T20:13:40,204 DEBUG [RS:0;jenkins-hbase3:33275 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-09-06T20:13:40,204 INFO [RS:0;jenkins-hbase3:33275 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-09-06T20:13:40,204 INFO [RS:0;jenkins-hbase3:33275 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-09-06T20:13:40,281 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-09-06T20:13:40,282 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236 2023-09-06T20:13:40,295 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:13:40,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-06T20:13:40,302 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-06T20:13:40,302 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:40,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:13:40,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-06T20:13:40,306 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-06T20:13:40,306 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:40,307 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:13:40,307 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-06T20:13:40,310 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-06T20:13:40,310 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:40,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:13:40,313 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740 2023-09-06T20:13:40,314 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740 2023-09-06T20:13:40,315 INFO [RS:0;jenkins-hbase3:33275 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-06T20:13:40,318 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-06T20:13:40,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-06T20:13:40,325 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:13:40,325 INFO [RS:0;jenkins-hbase3:33275 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C33275%2C1694031217268, suffix=, logDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/WALs/jenkins-hbase3.apache.org,33275,1694031217268, archiveDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/oldWALs, maxLogs=32 2023-09-06T20:13:40,326 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73105795, jitterRate=0.08936123549938202}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-06T20:13:40,329 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-06T20:13:40,329 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-06T20:13:40,329 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-06T20:13:40,329 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-06T20:13:40,329 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-06T20:13:40,329 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-06T20:13:40,331 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-06T20:13:40,331 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-06T20:13:40,337 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-09-06T20:13:40,337 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-09-06T20:13:40,346 DEBUG [RS:0;jenkins-hbase3:33275 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/WALs/jenkins-hbase3.apache.org,33275,1694031217268/jenkins-hbase3.apache.org%2C33275%2C1694031217268.1694031220327, exclude list is [], retry=0 2023-09-06T20:13:40,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-09-06T20:13:40,359 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42057,DS-38d94f8a-49ee-4b1d-9e2d-02a906c5d972,DISK] 2023-09-06T20:13:40,369 INFO [RS:0;jenkins-hbase3:33275 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/WALs/jenkins-hbase3.apache.org,33275,1694031217268/jenkins-hbase3.apache.org%2C33275%2C1694031217268.1694031220327 2023-09-06T20:13:40,370 DEBUG [RS:0;jenkins-hbase3:33275 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:42057,DS-38d94f8a-49ee-4b1d-9e2d-02a906c5d972,DISK]] 2023-09-06T20:13:40,373 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-09-06T20:13:40,378 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-09-06T20:13:40,531 DEBUG [jenkins-hbase3:35153 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-09-06T20:13:40,541 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:40,553 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,33275,1694031217268, state=OPENING 2023-09-06T20:13:40,564 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-09-06T20:13:40,567 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:13:40,567 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:13:40,568 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-09-06T20:13:40,568 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-09-06T20:13:40,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:13:40,770 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:40,773 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-09-06T20:13:40,778 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35916, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-09-06T20:13:40,794 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-09-06T20:13:40,794 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:13:40,795 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-09-06T20:13:40,799 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C33275%2C1694031217268.meta, suffix=.meta, logDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/WALs/jenkins-hbase3.apache.org,33275,1694031217268, archiveDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/oldWALs, maxLogs=32 2023-09-06T20:13:40,815 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/WALs/jenkins-hbase3.apache.org,33275,1694031217268/jenkins-hbase3.apache.org%2C33275%2C1694031217268.meta.1694031220800.meta, exclude list is [], retry=0 2023-09-06T20:13:40,821 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42057,DS-38d94f8a-49ee-4b1d-9e2d-02a906c5d972,DISK] 2023-09-06T20:13:40,834 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/WALs/jenkins-hbase3.apache.org,33275,1694031217268/jenkins-hbase3.apache.org%2C33275%2C1694031217268.meta.1694031220800.meta 2023-09-06T20:13:40,842 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:42057,DS-38d94f8a-49ee-4b1d-9e2d-02a906c5d972,DISK]] 2023-09-06T20:13:40,842 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:13:40,845 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-09-06T20:13:40,865 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-09-06T20:13:40,872 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-09-06T20:13:40,880 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-09-06T20:13:40,880 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:13:40,880 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-09-06T20:13:40,881 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-09-06T20:13:40,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-06T20:13:40,886 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-06T20:13:40,887 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:40,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:13:40,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-06T20:13:40,889 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-06T20:13:40,890 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:40,891 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:13:40,891 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-06T20:13:40,892 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-06T20:13:40,893 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:40,893 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:13:40,895 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740 2023-09-06T20:13:40,898 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740 2023-09-06T20:13:40,901 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-06T20:13:40,905 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-06T20:13:40,907 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75269822, jitterRate=0.12160775065422058}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-06T20:13:40,909 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-06T20:13:40,926 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1694031220758 2023-09-06T20:13:40,948 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:40,950 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-09-06T20:13:40,951 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-09-06T20:13:40,951 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,33275,1694031217268, state=OPEN 2023-09-06T20:13:40,954 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2023-09-06T20:13:40,954 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2023-09-06T20:13:40,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-09-06T20:13:40,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-09-06T20:13:40,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-09-06T20:13:40,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,33275,1694031217268 in 383 msec 2023-09-06T20:13:40,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-09-06T20:13:40,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 613 msec 2023-09-06T20:13:40,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.2710 sec 2023-09-06T20:13:40,975 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1694031220975, completionTime=-1 2023-09-06T20:13:40,976 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-09-06T20:13:40,976 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-09-06T20:13:41,054 DEBUG [hconnection-0x2779af75-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:13:41,058 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35930, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:13:41,077 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-09-06T20:13:41,077 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1694031281077 2023-09-06T20:13:41,077 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1694031341077 2023-09-06T20:13:41,077 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 101 msec 2023-09-06T20:13:41,118 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,35153,1694031215758-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:41,118 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,35153,1694031215758-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:41,118 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,35153,1694031215758-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:41,121 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:35153, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:41,121 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-09-06T20:13:41,128 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-09-06T20:13:41,138 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-09-06T20:13:41,142 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-06T20:13:41,153 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-09-06T20:13:41,157 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:13:41,159 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:41,162 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:13:41,186 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 46c263e9fd401abca49830e3d9e96c8e, NAME => 'hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236 2023-09-06T20:13:41,202 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:13:41,202 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 46c263e9fd401abca49830e3d9e96c8e, disabling compactions & flushes 2023-09-06T20:13:41,202 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e. 2023-09-06T20:13:41,203 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e. 2023-09-06T20:13:41,203 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e. after waiting 0 ms 2023-09-06T20:13:41,203 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e. 2023-09-06T20:13:41,203 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e. 2023-09-06T20:13:41,203 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 46c263e9fd401abca49830e3d9e96c8e: 2023-09-06T20:13:41,206 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:13:41,220 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1694031221208"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694031221208"}]},"ts":"1694031221208"} 2023-09-06T20:13:41,264 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:13:41,271 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:13:41,276 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031221271"}]},"ts":"1694031221271"} 2023-09-06T20:13:41,285 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-09-06T20:13:41,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=46c263e9fd401abca49830e3d9e96c8e, ASSIGN}] 2023-09-06T20:13:41,295 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=46c263e9fd401abca49830e3d9e96c8e, ASSIGN 2023-09-06T20:13:41,297 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=46c263e9fd401abca49830e3d9e96c8e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33275,1694031217268; forceNewPlan=false, retain=false 2023-09-06T20:13:41,448 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=46c263e9fd401abca49830e3d9e96c8e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:41,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 46c263e9fd401abca49830e3d9e96c8e, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:13:41,620 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e. 2023-09-06T20:13:41,621 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 46c263e9fd401abca49830e3d9e96c8e, NAME => 'hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:13:41,622 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 46c263e9fd401abca49830e3d9e96c8e 2023-09-06T20:13:41,622 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:13:41,622 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 46c263e9fd401abca49830e3d9e96c8e 2023-09-06T20:13:41,622 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 46c263e9fd401abca49830e3d9e96c8e 2023-09-06T20:13:41,625 INFO [StoreOpener-46c263e9fd401abca49830e3d9e96c8e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 46c263e9fd401abca49830e3d9e96c8e 2023-09-06T20:13:41,628 INFO [StoreOpener-46c263e9fd401abca49830e3d9e96c8e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46c263e9fd401abca49830e3d9e96c8e columnFamilyName info 2023-09-06T20:13:41,628 DEBUG [StoreOpener-46c263e9fd401abca49830e3d9e96c8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:41,629 INFO [StoreOpener-46c263e9fd401abca49830e3d9e96c8e-1 {}] regionserver.HStore(324): Store=46c263e9fd401abca49830e3d9e96c8e/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:13:41,631 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/namespace/46c263e9fd401abca49830e3d9e96c8e 2023-09-06T20:13:41,631 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/namespace/46c263e9fd401abca49830e3d9e96c8e 2023-09-06T20:13:41,635 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 46c263e9fd401abca49830e3d9e96c8e 2023-09-06T20:13:41,642 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/namespace/46c263e9fd401abca49830e3d9e96c8e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:13:41,643 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 46c263e9fd401abca49830e3d9e96c8e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66780945, jitterRate=-0.00488637387752533}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:13:41,644 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 46c263e9fd401abca49830e3d9e96c8e: 2023-09-06T20:13:41,647 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e., pid=6, masterSystemTime=1694031221613 2023-09-06T20:13:41,651 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e. 2023-09-06T20:13:41,651 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e. 2023-09-06T20:13:41,653 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=46c263e9fd401abca49830e3d9e96c8e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:41,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-09-06T20:13:41,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 46c263e9fd401abca49830e3d9e96c8e, server=jenkins-hbase3.apache.org,33275,1694031217268 in 200 msec 2023-09-06T20:13:41,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-09-06T20:13:41,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=46c263e9fd401abca49830e3d9e96c8e, ASSIGN in 369 msec 2023-09-06T20:13:41,670 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:13:41,671 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031221671"}]},"ts":"1694031221671"} 2023-09-06T20:13:41,675 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-09-06T20:13:41,678 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:13:41,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 534 msec 2023-09-06T20:13:41,757 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2023-09-06T20:13:41,760 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2023-09-06T20:13:41,760 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:13:41,760 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:13:41,817 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-09-06T20:13:41,837 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2023-09-06T20:13:41,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 41 msec 2023-09-06T20:13:41,864 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-09-06T20:13:41,880 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2023-09-06T20:13:41,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 27 msec 2023-09-06T20:13:41,903 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2023-09-06T20:13:41,909 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2023-09-06T20:13:41,910 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 4.542sec 2023-09-06T20:13:41,912 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-09-06T20:13:41,914 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-09-06T20:13:41,916 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-09-06T20:13:41,917 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-09-06T20:13:41,917 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-09-06T20:13:41,920 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,35153,1694031215758-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-06T20:13:41,920 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,35153,1694031215758-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-09-06T20:13:41,939 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x203b04e8 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:13:41,943 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-09-06T20:13:41,944 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-09-06T20:13:41,950 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dbd7947, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:13:41,956 DEBUG [Listener at localhost/35961 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-09-06T20:13:41,956 DEBUG [Listener at localhost/35961 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-09-06T20:13:41,972 DEBUG [hconnection-0x39f087ae-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:13:41,987 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35934, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:13:42,000 INFO [Listener at localhost/35961 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,35153,1694031215758 2023-09-06T20:13:42,027 INFO [Listener at localhost/35961 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testMixedAtomicity Thread=204, OpenFileDescriptor=478, MaxFileDescriptor=60000, SystemLoadAverage=480, ProcessCount=175, AvailableMemoryMB=5706 2023-09-06T20:13:42,048 DEBUG [Listener at localhost/35961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:13:42,066 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39298, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:13:42,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-06T20:13:42,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:13:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2023-09-06T20:13:42,106 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:13:42,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 9 2023-09-06T20:13:42,107 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:42,110 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:13:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:13:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:13:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:13:42,585 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236 2023-09-06T20:13:42,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:13:42,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1668): Closing a659f678dfa70a1da8b63a91043e10d5, disabling compactions & flushes 2023-09-06T20:13:42,628 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:42,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:42,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. after waiting 0 ms 2023-09-06T20:13:42,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:42,628 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:42,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1622): Region close journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:42,631 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:13:42,632 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1694031222631"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694031222631"}]},"ts":"1694031222631"} 2023-09-06T20:13:42,635 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:13:42,637 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:13:42,638 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031222637"}]},"ts":"1694031222637"} 2023-09-06T20:13:42,641 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2023-09-06T20:13:42,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=a659f678dfa70a1da8b63a91043e10d5, ASSIGN}] 2023-09-06T20:13:42,648 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=a659f678dfa70a1da8b63a91043e10d5, ASSIGN 2023-09-06T20:13:42,649 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a659f678dfa70a1da8b63a91043e10d5, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33275,1694031217268; forceNewPlan=false, retain=false 2023-09-06T20:13:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:13:42,802 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=a659f678dfa70a1da8b63a91043e10d5, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:42,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:13:42,970 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:42,970 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7254): Opening region: {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:13:42,971 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:42,971 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:13:42,971 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7296): checking encryption for a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:42,971 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7299): checking classloading for a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:42,974 INFO [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:42,978 INFO [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:13:42,979 INFO [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a659f678dfa70a1da8b63a91043e10d5 columnFamilyName A 2023-09-06T20:13:42,979 DEBUG [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:42,983 INFO [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] regionserver.HStore(324): Store=a659f678dfa70a1da8b63a91043e10d5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:13:42,983 INFO [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:42,986 INFO [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:13:42,986 INFO [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a659f678dfa70a1da8b63a91043e10d5 columnFamilyName B 2023-09-06T20:13:43,002 DEBUG [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:43,004 INFO [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] regionserver.HStore(324): Store=a659f678dfa70a1da8b63a91043e10d5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:13:43,004 INFO [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:43,008 INFO [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:13:43,009 INFO [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a659f678dfa70a1da8b63a91043e10d5 columnFamilyName C 2023-09-06T20:13:43,009 DEBUG [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:13:43,010 INFO [StoreOpener-a659f678dfa70a1da8b63a91043e10d5-1 {}] regionserver.HStore(324): Store=a659f678dfa70a1da8b63a91043e10d5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:13:43,011 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:43,014 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:43,015 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:43,022 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-06T20:13:43,027 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1076): writing seq id for a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:43,039 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:13:43,040 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): Opened a659f678dfa70a1da8b63a91043e10d5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62200713, jitterRate=-0.07313714921474457}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-06T20:13:43,042 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(992): Region open journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:43,045 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., pid=11, masterSystemTime=1694031222961 2023-09-06T20:13:43,048 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:43,049 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:43,050 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=a659f678dfa70a1da8b63a91043e10d5, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:43,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=11, resume processing ppid=10 2023-09-06T20:13:43,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 in 246 msec 2023-09-06T20:13:43,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=10, resume processing ppid=9 2023-09-06T20:13:43,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=a659f678dfa70a1da8b63a91043e10d5, ASSIGN in 412 msec 2023-09-06T20:13:43,063 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:13:43,063 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031223063"}]},"ts":"1694031223063"} 2023-09-06T20:13:43,066 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2023-09-06T20:13:43,072 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:13:43,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 974 msec 2023-09-06T20:13:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:13:43,243 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 9 completed 2023-09-06T20:13:43,249 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6d771096 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:13:43,258 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58420f36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:13:43,261 DEBUG [Listener at localhost/35961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:13:43,265 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35938, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:13:43,269 DEBUG [Listener at localhost/35961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:13:43,272 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39300, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:13:43,283 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3bd38762 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:13:43,291 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17d42cac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:13:43,293 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4c616450 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:13:43,301 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29e2bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:13:43,303 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1fb807fe to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:13:43,315 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c717a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:13:43,317 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2be9788f to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:13:43,325 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1341b5a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:13:43,327 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x77c03710 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:13:43,379 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b6222d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:13:43,382 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x11920f5c to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:13:43,386 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7423dfd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:13:43,388 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x190efedc to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:13:43,399 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1867fa4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:13:43,400 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x57847d41 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:13:43,409 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70965875, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:13:43,411 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2d7329e4 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:13:43,420 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e9377e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:13:43,440 DEBUG [hconnection-0x785ef273-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:13:43,440 DEBUG [hconnection-0x302728ba-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:13:43,442 DEBUG [hconnection-0x70e19e69-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:13:43,449 DEBUG [hconnection-0x7e71efc1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:13:43,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:13:43,460 DEBUG [hconnection-0x5a4e7cd7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:13:43,461 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35968, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:13:43,461 DEBUG [hconnection-0x16d2a737-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:13:43,463 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35952, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:13:43,470 DEBUG [hconnection-0x36d0c8ae-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:13:43,470 DEBUG [hconnection-0x1d7239a1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:13:43,472 DEBUG [hconnection-0x543be622-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:13:43,476 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35984, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:13:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees 2023-09-06T20:13:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:13:43,491 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35998, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:13:43,491 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35992, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:13:43,495 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:13:43,499 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=12, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:13:43,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:13:43,511 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36012, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:13:43,520 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36016, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:13:43,520 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36020, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:13:43,528 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36022, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:13:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:13:43,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:13:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:43,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:43,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:43,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:43,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:43,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:43,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:43,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2023-09-06T20:13:43,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:43,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:43,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:43,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:43,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:13:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:43,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031283779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:43,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031283795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:43,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031283798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:43,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031283800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:43,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031283803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:43,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/81f21d3db4e64aefa74de166313ffa6b is 50, key is test_row_0/A:col10/1694031223634/Put/seqid=0 2023-09-06T20:13:43,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031283924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:43,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031283924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:43,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031283926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:43,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031283926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:43,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031283927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:43,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2023-09-06T20:13:43,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:43,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:43,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:43,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:43,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:13:44,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2023-09-06T20:13:44,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031284131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:44,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031284133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:44,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031284139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:44,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031284139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:44,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031284142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:44,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:44,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:44,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:44,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:44,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:44,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2023-09-06T20:13:44,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:44,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:44,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:44,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:44,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:44,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/81f21d3db4e64aefa74de166313ffa6b 2023-09-06T20:13:44,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031284438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:44,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031284439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:44,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:44,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031284446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031284446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:44,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031284451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:44,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2023-09-06T20:13:44,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:44,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:44,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:44,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:44,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:44,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/6bb0560e44d04290aeb4945b301f5f1b is 50, key is test_row_0/B:col10/1694031223634/Put/seqid=0 2023-09-06T20:13:44,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/6bb0560e44d04290aeb4945b301f5f1b 2023-09-06T20:13:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:13:44,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/d120dc0fc60b4bbca21dc4c60ade4628 is 50, key is test_row_0/C:col10/1694031223634/Put/seqid=0 2023-09-06T20:13:44,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2023-09-06T20:13:44,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:44,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:44,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:44,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:44,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/d120dc0fc60b4bbca21dc4c60ade4628 2023-09-06T20:13:44,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/81f21d3db4e64aefa74de166313ffa6b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/81f21d3db4e64aefa74de166313ffa6b 2023-09-06T20:13:44,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/81f21d3db4e64aefa74de166313ffa6b, entries=150, sequenceid=15, filesize=11.7 K 2023-09-06T20:13:44,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/6bb0560e44d04290aeb4945b301f5f1b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/6bb0560e44d04290aeb4945b301f5f1b 2023-09-06T20:13:44,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/6bb0560e44d04290aeb4945b301f5f1b, entries=150, sequenceid=15, filesize=11.7 K 2023-09-06T20:13:44,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/d120dc0fc60b4bbca21dc4c60ade4628 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d120dc0fc60b4bbca21dc4c60ade4628 2023-09-06T20:13:44,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d120dc0fc60b4bbca21dc4c60ade4628, entries=150, sequenceid=15, filesize=11.7 K 2023-09-06T20:13:44,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a659f678dfa70a1da8b63a91043e10d5 in 1135ms, sequenceid=15, compaction requested=false 2023-09-06T20:13:44,775 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2023-09-06T20:13:44,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:44,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2023-09-06T20:13:44,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:44,813 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2023-09-06T20:13:44,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:44,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:44,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:44,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:44,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:44,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:44,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/6ee081991ab94dc5b4d4d4feed7f0a1f is 50, key is test_row_0/A:col10/1694031223750/Put/seqid=0 2023-09-06T20:13:44,865 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/6ee081991ab94dc5b4d4d4feed7f0a1f 2023-09-06T20:13:44,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/03486942346a46329b8297f9f499f596 is 50, key is test_row_0/B:col10/1694031223750/Put/seqid=0 2023-09-06T20:13:44,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:44,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:44,980 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/03486942346a46329b8297f9f499f596 2023-09-06T20:13:45,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031284997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031284997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031284999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031285008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031285009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/3d5e625628264082b0ed42b3a3fc82d3 is 50, key is test_row_0/C:col10/1694031223750/Put/seqid=0 2023-09-06T20:13:45,082 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/3d5e625628264082b0ed42b3a3fc82d3 2023-09-06T20:13:45,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/6ee081991ab94dc5b4d4d4feed7f0a1f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/6ee081991ab94dc5b4d4d4feed7f0a1f 2023-09-06T20:13:45,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031285113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,117 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/6ee081991ab94dc5b4d4d4feed7f0a1f, entries=150, sequenceid=37, filesize=11.7 K 2023-09-06T20:13:45,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/03486942346a46329b8297f9f499f596 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/03486942346a46329b8297f9f499f596 2023-09-06T20:13:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031285113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031285113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031285114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031285115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,149 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/03486942346a46329b8297f9f499f596, entries=150, sequenceid=37, filesize=11.7 K 2023-09-06T20:13:45,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/3d5e625628264082b0ed42b3a3fc82d3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3d5e625628264082b0ed42b3a3fc82d3 2023-09-06T20:13:45,165 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3d5e625628264082b0ed42b3a3fc82d3, entries=150, sequenceid=37, filesize=11.7 K 2023-09-06T20:13:45,168 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(3022): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for a659f678dfa70a1da8b63a91043e10d5 in 355ms, sequenceid=37, compaction requested=false 2023-09-06T20:13:45,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:45,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:45,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=13 2023-09-06T20:13:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=13 2023-09-06T20:13:45,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=13, resume processing ppid=12 2023-09-06T20:13:45,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, ppid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6710 sec 2023-09-06T20:13:45,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees in 1.7150 sec 2023-09-06T20:13:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:45,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-06T20:13:45,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:45,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:45,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:45,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:45,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:45,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:45,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/63fcd3ca58bc4d109807677a646de71e is 50, key is test_row_0/A:col10/1694031225322/Put/seqid=0 2023-09-06T20:13:45,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031285363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031285366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031285367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031285369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031285364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/63fcd3ca58bc4d109807677a646de71e 2023-09-06T20:13:45,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/b5dda693d5db4638a0f78546cd5520f4 is 50, key is test_row_0/B:col10/1694031225322/Put/seqid=0 2023-09-06T20:13:45,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031285472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031285476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031285477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031285476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031285491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:13:45,598 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 12 completed 2023-09-06T20:13:45,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:13:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees 2023-09-06T20:13:45,608 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:13:45,609 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=14, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:13:45,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:13:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=14 2023-09-06T20:13:45,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031285680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031285683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031285686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031285685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031285704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=14 2023-09-06T20:13:45,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2023-09-06T20:13:45,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:45,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:45,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:45,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(53): pid=15 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:45,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=14 2023-09-06T20:13:45,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2023-09-06T20:13:45,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:45,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:45,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:45,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/b5dda693d5db4638a0f78546cd5520f4 2023-09-06T20:13:45,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(53): pid=15 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:45,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:45,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031285990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031285991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031285994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:45,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031285995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/b2b0f2c4955042a6a5bd64c6ea6dc36f is 50, key is test_row_0/C:col10/1694031225322/Put/seqid=0 2023-09-06T20:13:46,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031286012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/b2b0f2c4955042a6a5bd64c6ea6dc36f 2023-09-06T20:13:46,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2023-09-06T20:13:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:46,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/63fcd3ca58bc4d109807677a646de71e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/63fcd3ca58bc4d109807677a646de71e 2023-09-06T20:13:46,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(53): pid=15 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:46,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:46,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/63fcd3ca58bc4d109807677a646de71e, entries=150, sequenceid=52, filesize=11.7 K 2023-09-06T20:13:46,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/b5dda693d5db4638a0f78546cd5520f4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b5dda693d5db4638a0f78546cd5520f4 2023-09-06T20:13:46,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b5dda693d5db4638a0f78546cd5520f4, entries=150, sequenceid=52, filesize=11.7 K 2023-09-06T20:13:46,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/b2b0f2c4955042a6a5bd64c6ea6dc36f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b2b0f2c4955042a6a5bd64c6ea6dc36f 2023-09-06T20:13:46,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b2b0f2c4955042a6a5bd64c6ea6dc36f, entries=150, sequenceid=52, filesize=11.7 K 2023-09-06T20:13:46,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for a659f678dfa70a1da8b63a91043e10d5 in 853ms, sequenceid=52, compaction requested=true 2023-09-06T20:13:46,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:46,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:13:46,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:46,186 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:46,186 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:46,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:13:46,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:46,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:46,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:46,193 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:46,195 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/B is initiating minor compaction (all files) 2023-09-06T20:13:46,195 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/B in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:46,195 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/6bb0560e44d04290aeb4945b301f5f1b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/03486942346a46329b8297f9f499f596, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b5dda693d5db4638a0f78546cd5520f4] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=35.1 K 2023-09-06T20:13:46,199 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 6bb0560e44d04290aeb4945b301f5f1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1694031223590 2023-09-06T20:13:46,200 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 03486942346a46329b8297f9f499f596, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1694031223750 2023-09-06T20:13:46,200 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b5dda693d5db4638a0f78546cd5520f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1694031225005 2023-09-06T20:13:46,202 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:46,202 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/A is initiating minor compaction (all files) 2023-09-06T20:13:46,202 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/A in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:46,202 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/81f21d3db4e64aefa74de166313ffa6b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/6ee081991ab94dc5b4d4d4feed7f0a1f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/63fcd3ca58bc4d109807677a646de71e] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=35.1 K 2023-09-06T20:13:46,207 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 81f21d3db4e64aefa74de166313ffa6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1694031223590 2023-09-06T20:13:46,211 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6ee081991ab94dc5b4d4d4feed7f0a1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1694031223750 2023-09-06T20:13:46,213 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 63fcd3ca58bc4d109807677a646de71e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1694031225005 2023-09-06T20:13:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=14 2023-09-06T20:13:46,253 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#A#compaction#9 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:46,254 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/642fe0fe86a040fba5ca9046fd2d923f is 50, key is test_row_0/A:col10/1694031225322/Put/seqid=0 2023-09-06T20:13:46,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2023-09-06T20:13:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:46,270 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#B#compaction#10 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:46,270 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2023-09-06T20:13:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:46,271 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/abeff01a37ae465b8058a323135f094b is 50, key is test_row_0/B:col10/1694031225322/Put/seqid=0 2023-09-06T20:13:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:46,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:46,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:46,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:46,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/603c67e5f3f64a4087dbea4de88a79a6 is 50, key is test_row_0/A:col10/1694031225366/Put/seqid=0 2023-09-06T20:13:46,408 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/642fe0fe86a040fba5ca9046fd2d923f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/642fe0fe86a040fba5ca9046fd2d923f 2023-09-06T20:13:46,450 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/A of a659f678dfa70a1da8b63a91043e10d5 into 642fe0fe86a040fba5ca9046fd2d923f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:46,450 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:46,450 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/A, priority=13, startTime=1694031226180; duration=0sec 2023-09-06T20:13:46,451 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:46,451 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:46,451 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:46,471 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:46,472 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/C is initiating minor compaction (all files) 2023-09-06T20:13:46,475 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/C in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:46,475 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d120dc0fc60b4bbca21dc4c60ade4628, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3d5e625628264082b0ed42b3a3fc82d3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b2b0f2c4955042a6a5bd64c6ea6dc36f] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=35.1 K 2023-09-06T20:13:46,476 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting d120dc0fc60b4bbca21dc4c60ade4628, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1694031223590 2023-09-06T20:13:46,477 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3d5e625628264082b0ed42b3a3fc82d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1694031223750 2023-09-06T20:13:46,480 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting b2b0f2c4955042a6a5bd64c6ea6dc36f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1694031225005 2023-09-06T20:13:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:46,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:46,531 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#C#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:46,533 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/15dbdd8c79da4cfdb396aa9099d05077 is 50, key is test_row_0/C:col10/1694031225322/Put/seqid=0 2023-09-06T20:13:46,543 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-06T20:13:46,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031286528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031286534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031286543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031286544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031286544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,646 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-09-06T20:13:46,647 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-09-06T20:13:46,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031286661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031286662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031286669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031286670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=14 2023-09-06T20:13:46,788 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/603c67e5f3f64a4087dbea4de88a79a6 2023-09-06T20:13:46,808 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/abeff01a37ae465b8058a323135f094b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/abeff01a37ae465b8058a323135f094b 2023-09-06T20:13:46,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/b999c41508784c859fa517dbe5a42f2d is 50, key is test_row_0/B:col10/1694031225366/Put/seqid=0 2023-09-06T20:13:46,832 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/B of a659f678dfa70a1da8b63a91043e10d5 into abeff01a37ae465b8058a323135f094b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:46,832 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:46,832 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/B, priority=13, startTime=1694031226186; duration=0sec 2023-09-06T20:13:46,832 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:46,832 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:46,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:46,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031286866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031286867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:46,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031286879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031286880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:46,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-09-06T20:13:46,890 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-09-06T20:13:46,892 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-09-06T20:13:46,892 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-09-06T20:13:46,894 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:13:46,894 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-09-06T20:13:46,894 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-09-06T20:13:46,894 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-09-06T20:13:46,896 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2023-09-06T20:13:46,896 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2023-09-06T20:13:46,992 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/15dbdd8c79da4cfdb396aa9099d05077 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/15dbdd8c79da4cfdb396aa9099d05077 2023-09-06T20:13:47,004 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/C of a659f678dfa70a1da8b63a91043e10d5 into 15dbdd8c79da4cfdb396aa9099d05077(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:47,005 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:47,005 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/C, priority=13, startTime=1694031226187; duration=0sec 2023-09-06T20:13:47,005 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:47,005 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:47,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031287176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:47,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031287176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:47,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031287183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:47,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:47,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031287186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:47,265 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/b999c41508784c859fa517dbe5a42f2d 2023-09-06T20:13:47,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/d687fa405c1a4304a7948e13556cf1e9 is 50, key is test_row_0/C:col10/1694031225366/Put/seqid=0 2023-09-06T20:13:47,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031287558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:47,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031287681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031287683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:47,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031287688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:47,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031287691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=14 2023-09-06T20:13:47,742 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/d687fa405c1a4304a7948e13556cf1e9 2023-09-06T20:13:47,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/603c67e5f3f64a4087dbea4de88a79a6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/603c67e5f3f64a4087dbea4de88a79a6 2023-09-06T20:13:47,768 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/603c67e5f3f64a4087dbea4de88a79a6, entries=150, sequenceid=73, filesize=11.7 K 2023-09-06T20:13:47,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/b999c41508784c859fa517dbe5a42f2d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b999c41508784c859fa517dbe5a42f2d 2023-09-06T20:13:47,801 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b999c41508784c859fa517dbe5a42f2d, entries=150, sequenceid=73, filesize=11.7 K 2023-09-06T20:13:47,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/d687fa405c1a4304a7948e13556cf1e9 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d687fa405c1a4304a7948e13556cf1e9 2023-09-06T20:13:47,815 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d687fa405c1a4304a7948e13556cf1e9, entries=150, sequenceid=73, filesize=11.7 K 2023-09-06T20:13:47,821 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(3022): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for a659f678dfa70a1da8b63a91043e10d5 in 1551ms, sequenceid=73, compaction requested=false 2023-09-06T20:13:47,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:47,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:47,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=15 2023-09-06T20:13:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=15 2023-09-06T20:13:47,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-09-06T20:13:47,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2150 sec 2023-09-06T20:13:47,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees in 2.2250 sec 2023-09-06T20:13:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:48,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2023-09-06T20:13:48,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:48,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:48,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:48,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:48,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:48,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:48,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/f9914ec9c7be456fb1a22e10b0ed09a7 is 50, key is test_row_0/A:col10/1694031228692/Put/seqid=0 2023-09-06T20:13:48,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/f9914ec9c7be456fb1a22e10b0ed09a7 2023-09-06T20:13:48,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031288741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:48,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031288742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:48,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031288749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:48,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:48,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031288751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:48,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/57467e829d774c89be8ae593ecceafa1 is 50, key is test_row_0/B:col10/1694031228692/Put/seqid=0 2023-09-06T20:13:48,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:48,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031288852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031288852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:48,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031288856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:48,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031288856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:49,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031289055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:49,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031289055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:49,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031289059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:49,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031289060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:49,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/57467e829d774c89be8ae593ecceafa1 2023-09-06T20:13:49,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/c125f6851bcb475fb058277cfdd13659 is 50, key is test_row_0/C:col10/1694031228692/Put/seqid=0 2023-09-06T20:13:49,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:49,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031289358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:49,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:49,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031289358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:49,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:49,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031289363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:49,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:49,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031289366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:49,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031289565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:49,568 DEBUG [Thread-146 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4202 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., hostname=jenkins-hbase3.apache.org,33275,1694031217268, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:130) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-06T20:13:49,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/c125f6851bcb475fb058277cfdd13659 2023-09-06T20:13:49,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/f9914ec9c7be456fb1a22e10b0ed09a7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/f9914ec9c7be456fb1a22e10b0ed09a7 2023-09-06T20:13:49,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/f9914ec9c7be456fb1a22e10b0ed09a7, entries=150, sequenceid=92, filesize=11.7 K 2023-09-06T20:13:49,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/57467e829d774c89be8ae593ecceafa1 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/57467e829d774c89be8ae593ecceafa1 2023-09-06T20:13:49,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/57467e829d774c89be8ae593ecceafa1, entries=150, sequenceid=92, filesize=11.7 K 2023-09-06T20:13:49,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/c125f6851bcb475fb058277cfdd13659 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/c125f6851bcb475fb058277cfdd13659 2023-09-06T20:13:49,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/c125f6851bcb475fb058277cfdd13659, entries=150, sequenceid=92, filesize=11.7 K 2023-09-06T20:13:49,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for a659f678dfa70a1da8b63a91043e10d5 in 984ms, sequenceid=92, compaction requested=true 2023-09-06T20:13:49,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:49,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:13:49,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:49,686 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:49,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:13:49,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:49,687 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:49,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:49,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:49,690 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:49,690 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/A is initiating minor compaction (all files) 2023-09-06T20:13:49,690 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/A in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:49,690 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:49,690 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/642fe0fe86a040fba5ca9046fd2d923f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/603c67e5f3f64a4087dbea4de88a79a6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/f9914ec9c7be456fb1a22e10b0ed09a7] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=35.2 K 2023-09-06T20:13:49,690 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/B is initiating minor compaction (all files) 2023-09-06T20:13:49,690 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/B in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:49,690 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/abeff01a37ae465b8058a323135f094b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b999c41508784c859fa517dbe5a42f2d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/57467e829d774c89be8ae593ecceafa1] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=35.2 K 2023-09-06T20:13:49,691 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 642fe0fe86a040fba5ca9046fd2d923f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1694031225005 2023-09-06T20:13:49,691 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting abeff01a37ae465b8058a323135f094b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1694031225005 2023-09-06T20:13:49,691 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 603c67e5f3f64a4087dbea4de88a79a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1694031225362 2023-09-06T20:13:49,692 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b999c41508784c859fa517dbe5a42f2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1694031225362 2023-09-06T20:13:49,692 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting f9914ec9c7be456fb1a22e10b0ed09a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1694031226525 2023-09-06T20:13:49,694 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 57467e829d774c89be8ae593ecceafa1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1694031226525 2023-09-06T20:13:49,715 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:49,715 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:49,716 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/699ac0859363472cab95ea23b9d07743 is 50, key is test_row_0/B:col10/1694031228692/Put/seqid=0 2023-09-06T20:13:49,716 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/e417bfcb98734ce08aca3ea43b2f8218 is 50, key is test_row_0/A:col10/1694031228692/Put/seqid=0 2023-09-06T20:13:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=14 2023-09-06T20:13:49,728 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 14 completed 2023-09-06T20:13:49,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:13:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees 2023-09-06T20:13:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:13:49,754 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:13:49,756 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=16, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:13:49,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:13:49,780 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/699ac0859363472cab95ea23b9d07743 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/699ac0859363472cab95ea23b9d07743 2023-09-06T20:13:49,791 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/B of a659f678dfa70a1da8b63a91043e10d5 into 699ac0859363472cab95ea23b9d07743(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:49,792 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:49,792 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/B, priority=13, startTime=1694031229687; duration=0sec 2023-09-06T20:13:49,792 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:49,792 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:49,792 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:49,794 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:49,794 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/C is initiating minor compaction (all files) 2023-09-06T20:13:49,794 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/C in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:49,795 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/15dbdd8c79da4cfdb396aa9099d05077, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d687fa405c1a4304a7948e13556cf1e9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/c125f6851bcb475fb058277cfdd13659] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=35.2 K 2023-09-06T20:13:49,796 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 15dbdd8c79da4cfdb396aa9099d05077, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1694031225005 2023-09-06T20:13:49,796 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting d687fa405c1a4304a7948e13556cf1e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1694031225362 2023-09-06T20:13:49,797 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting c125f6851bcb475fb058277cfdd13659, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1694031226525 2023-09-06T20:13:49,820 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#C#compaction#20 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:49,821 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/adaca99d26424096bc5f54516ffd1f3d is 50, key is test_row_0/C:col10/1694031228692/Put/seqid=0 2023-09-06T20:13:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:13:49,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2023-09-06T20:13:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:49,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:49,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:49,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:49,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:49,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:49,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:49,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/2938d16a54e644019d0d8407b7d8da0b is 50, key is test_row_0/A:col10/1694031229866/Put/seqid=0 2023-09-06T20:13:49,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031289892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:49,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031289900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:49,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031289900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:49,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031289902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:49,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2023-09-06T20:13:49,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:49,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:49,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:49,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] handler.RSProcedureHandler(53): pid=17 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:49,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=17 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=17 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031290001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:50,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031290005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:50,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031290005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:50,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031290012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:13:50,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2023-09-06T20:13:50,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:50,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:50,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:50,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] handler.RSProcedureHandler(53): pid=17 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=17 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=17 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,184 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/e417bfcb98734ce08aca3ea43b2f8218 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/e417bfcb98734ce08aca3ea43b2f8218 2023-09-06T20:13:50,194 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/A of a659f678dfa70a1da8b63a91043e10d5 into e417bfcb98734ce08aca3ea43b2f8218(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:50,194 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:50,194 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/A, priority=13, startTime=1694031229686; duration=0sec 2023-09-06T20:13:50,194 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:50,195 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:50,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:50,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031290204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:50,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031290210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:50,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031290210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:50,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:50,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031290216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:50,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2023-09-06T20:13:50,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:50,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:50,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:50,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] handler.RSProcedureHandler(53): pid=17 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=17 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=17 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,247 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/adaca99d26424096bc5f54516ffd1f3d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/adaca99d26424096bc5f54516ffd1f3d 2023-09-06T20:13:50,257 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/C of a659f678dfa70a1da8b63a91043e10d5 into adaca99d26424096bc5f54516ffd1f3d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:50,257 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:50,257 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/C, priority=13, startTime=1694031229687; duration=0sec 2023-09-06T20:13:50,258 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:50,258 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:50,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/2938d16a54e644019d0d8407b7d8da0b 2023-09-06T20:13:50,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/9b939b787fc54e6086d1ce6ebbc75cb6 is 50, key is test_row_0/B:col10/1694031229866/Put/seqid=0 2023-09-06T20:13:50,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:13:50,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/9b939b787fc54e6086d1ce6ebbc75cb6 2023-09-06T20:13:50,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/f2b1010641a94035a1e9b582fb48d607 is 50, key is test_row_0/C:col10/1694031229866/Put/seqid=0 2023-09-06T20:13:50,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2023-09-06T20:13:50,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:50,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:50,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:50,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] handler.RSProcedureHandler(53): pid=17 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=17 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=17 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031290510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:50,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031290516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:50,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031290516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:50,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031290520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:50,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2023-09-06T20:13:50,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:50,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:50,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] handler.RSProcedureHandler(53): pid=17 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=17 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=17 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2023-09-06T20:13:50,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:50,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:50,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:50,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] handler.RSProcedureHandler(53): pid=17 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=17 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=17 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:50,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/f2b1010641a94035a1e9b582fb48d607 2023-09-06T20:13:50,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/2938d16a54e644019d0d8407b7d8da0b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2938d16a54e644019d0d8407b7d8da0b 2023-09-06T20:13:50,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2938d16a54e644019d0d8407b7d8da0b, entries=150, sequenceid=116, filesize=11.7 K 2023-09-06T20:13:50,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/9b939b787fc54e6086d1ce6ebbc75cb6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/9b939b787fc54e6086d1ce6ebbc75cb6 2023-09-06T20:13:50,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/9b939b787fc54e6086d1ce6ebbc75cb6, entries=150, sequenceid=116, filesize=11.7 K 2023-09-06T20:13:50,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/f2b1010641a94035a1e9b582fb48d607 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/f2b1010641a94035a1e9b582fb48d607 2023-09-06T20:13:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:13:50,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/f2b1010641a94035a1e9b582fb48d607, entries=150, sequenceid=116, filesize=11.7 K 2023-09-06T20:13:50,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for a659f678dfa70a1da8b63a91043e10d5 in 989ms, sequenceid=116, compaction requested=false 2023-09-06T20:13:50,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:50,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2023-09-06T20:13:50,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:50,863 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2023-09-06T20:13:50,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:50,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:50,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:50,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:50,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:50,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:50,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/83b8ad16f2d946319994542da94c178d is 50, key is test_row_0/A:col10/1694031229895/Put/seqid=0 2023-09-06T20:13:50,924 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/83b8ad16f2d946319994542da94c178d 2023-09-06T20:13:50,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/14265a98310f4dc1b384467dd9336c2b is 50, key is test_row_0/B:col10/1694031229895/Put/seqid=0 2023-09-06T20:13:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:51,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:51,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031291058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031291058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031291059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031291061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031291163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031291164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031291164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031291165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,350 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/14265a98310f4dc1b384467dd9336c2b 2023-09-06T20:13:51,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/055dcc36f1bf4a098b83812ac123c66c is 50, key is test_row_0/C:col10/1694031229895/Put/seqid=0 2023-09-06T20:13:51,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031291366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031291369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031291369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031291370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031291671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031291673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031291676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031291676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:51,782 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/055dcc36f1bf4a098b83812ac123c66c 2023-09-06T20:13:51,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/83b8ad16f2d946319994542da94c178d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/83b8ad16f2d946319994542da94c178d 2023-09-06T20:13:51,800 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/83b8ad16f2d946319994542da94c178d, entries=150, sequenceid=132, filesize=11.8 K 2023-09-06T20:13:51,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/14265a98310f4dc1b384467dd9336c2b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/14265a98310f4dc1b384467dd9336c2b 2023-09-06T20:13:51,814 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/14265a98310f4dc1b384467dd9336c2b, entries=150, sequenceid=132, filesize=11.8 K 2023-09-06T20:13:51,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/055dcc36f1bf4a098b83812ac123c66c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/055dcc36f1bf4a098b83812ac123c66c 2023-09-06T20:13:51,832 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/055dcc36f1bf4a098b83812ac123c66c, entries=150, sequenceid=132, filesize=11.8 K 2023-09-06T20:13:51,834 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(3022): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a659f678dfa70a1da8b63a91043e10d5 in 971ms, sequenceid=132, compaction requested=true 2023-09-06T20:13:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=17 2023-09-06T20:13:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=17 2023-09-06T20:13:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:13:51,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-09-06T20:13:51,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0860 sec 2023-09-06T20:13:51,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees in 2.1160 sec 2023-09-06T20:13:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:52,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-06T20:13:52,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:52,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:52,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:52,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:52,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:52,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:52,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/c8af0a32c2d3462dabda3417035c9dfe is 50, key is test_row_0/A:col10/1694031231051/Put/seqid=0 2023-09-06T20:13:52,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031292203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031292203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031292207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031292208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/c8af0a32c2d3462dabda3417035c9dfe 2023-09-06T20:13:52,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/4e22c69bb7fe41babbc91b5c00660619 is 50, key is test_row_0/B:col10/1694031231051/Put/seqid=0 2023-09-06T20:13:52,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031292327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031292327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031292327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/4e22c69bb7fe41babbc91b5c00660619 2023-09-06T20:13:52,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031292333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/44786d4c24804e3da0a523bc557c145d is 50, key is test_row_0/C:col10/1694031231051/Put/seqid=0 2023-09-06T20:13:52,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/44786d4c24804e3da0a523bc557c145d 2023-09-06T20:13:52,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/c8af0a32c2d3462dabda3417035c9dfe as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c8af0a32c2d3462dabda3417035c9dfe 2023-09-06T20:13:52,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c8af0a32c2d3462dabda3417035c9dfe, entries=150, sequenceid=156, filesize=11.8 K 2023-09-06T20:13:52,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/4e22c69bb7fe41babbc91b5c00660619 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/4e22c69bb7fe41babbc91b5c00660619 2023-09-06T20:13:52,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/4e22c69bb7fe41babbc91b5c00660619, entries=150, sequenceid=156, filesize=11.8 K 2023-09-06T20:13:52,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/44786d4c24804e3da0a523bc557c145d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/44786d4c24804e3da0a523bc557c145d 2023-09-06T20:13:52,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/44786d4c24804e3da0a523bc557c145d, entries=150, sequenceid=156, filesize=11.8 K 2023-09-06T20:13:52,556 INFO [AsyncFSWAL-0-hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236 {}] wal.AbstractFSWAL(1174): Slow sync cost: 158 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42057,DS-38d94f8a-49ee-4b1d-9e2d-02a906c5d972,DISK]] 2023-09-06T20:13:52,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for a659f678dfa70a1da8b63a91043e10d5 in 374ms, sequenceid=156, compaction requested=true 2023-09-06T20:13:52,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:52,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:13:52,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:52,557 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:13:52,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:13:52,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:52,557 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:13:52,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:52,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:52,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2023-09-06T20:13:52,572 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:13:52,572 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/A is initiating minor compaction (all files) 2023-09-06T20:13:52,572 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/A in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:52,572 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/e417bfcb98734ce08aca3ea43b2f8218, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2938d16a54e644019d0d8407b7d8da0b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/83b8ad16f2d946319994542da94c178d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c8af0a32c2d3462dabda3417035c9dfe] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=47.2 K 2023-09-06T20:13:52,572 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:13:52,572 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/B is initiating minor compaction (all files) 2023-09-06T20:13:52,573 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/B in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:52,573 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/699ac0859363472cab95ea23b9d07743, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/9b939b787fc54e6086d1ce6ebbc75cb6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/14265a98310f4dc1b384467dd9336c2b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/4e22c69bb7fe41babbc91b5c00660619] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=47.2 K 2023-09-06T20:13:52,573 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting e417bfcb98734ce08aca3ea43b2f8218, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1694031226525 2023-09-06T20:13:52,574 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 699ac0859363472cab95ea23b9d07743, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1694031226525 2023-09-06T20:13:52,574 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2938d16a54e644019d0d8407b7d8da0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1694031228725 2023-09-06T20:13:52,574 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 9b939b787fc54e6086d1ce6ebbc75cb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1694031228725 2023-09-06T20:13:52,575 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 83b8ad16f2d946319994542da94c178d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1694031229885 2023-09-06T20:13:52,575 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 14265a98310f4dc1b384467dd9336c2b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1694031229885 2023-09-06T20:13:52,575 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting c8af0a32c2d3462dabda3417035c9dfe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1694031231051 2023-09-06T20:13:52,576 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 4e22c69bb7fe41babbc91b5c00660619, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1694031231051 2023-09-06T20:13:52,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:52,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:52,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:52,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:52,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:52,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:52,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/d8ababdfea1e4340b03070dbf4e3aaab is 50, key is test_row_0/A:col10/1694031232561/Put/seqid=0 2023-09-06T20:13:52,605 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#B#compaction#31 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:52,605 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#A#compaction#32 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:52,606 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/9977f7217d5a46669317b16af6a035b4 is 50, key is test_row_0/B:col10/1694031231051/Put/seqid=0 2023-09-06T20:13:52,606 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/06bafdf833464e04bd8a1aebbdceae4b is 50, key is test_row_0/A:col10/1694031231051/Put/seqid=0 2023-09-06T20:13:52,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031292611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031292615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031292615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031292616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/d8ababdfea1e4340b03070dbf4e3aaab 2023-09-06T20:13:52,668 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/06bafdf833464e04bd8a1aebbdceae4b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/06bafdf833464e04bd8a1aebbdceae4b 2023-09-06T20:13:52,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/185bbf9fb325422ab5e56a4c9904dfa7 is 50, key is test_row_0/B:col10/1694031232561/Put/seqid=0 2023-09-06T20:13:52,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/185bbf9fb325422ab5e56a4c9904dfa7 2023-09-06T20:13:52,691 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/A of a659f678dfa70a1da8b63a91043e10d5 into 06bafdf833464e04bd8a1aebbdceae4b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:52,692 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:52,692 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/A, priority=12, startTime=1694031232557; duration=0sec 2023-09-06T20:13:52,692 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:52,692 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:52,692 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:13:52,709 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/9977f7217d5a46669317b16af6a035b4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/9977f7217d5a46669317b16af6a035b4 2023-09-06T20:13:52,709 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:13:52,709 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/C is initiating minor compaction (all files) 2023-09-06T20:13:52,709 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/C in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:52,709 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/adaca99d26424096bc5f54516ffd1f3d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/f2b1010641a94035a1e9b582fb48d607, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/055dcc36f1bf4a098b83812ac123c66c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/44786d4c24804e3da0a523bc557c145d] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=47.2 K 2023-09-06T20:13:52,710 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting adaca99d26424096bc5f54516ffd1f3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1694031226525 2023-09-06T20:13:52,712 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting f2b1010641a94035a1e9b582fb48d607, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1694031228725 2023-09-06T20:13:52,713 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 055dcc36f1bf4a098b83812ac123c66c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1694031229885 2023-09-06T20:13:52,714 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 44786d4c24804e3da0a523bc557c145d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1694031231051 2023-09-06T20:13:52,719 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/B of a659f678dfa70a1da8b63a91043e10d5 into 9977f7217d5a46669317b16af6a035b4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:52,719 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:52,719 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/B, priority=12, startTime=1694031232557; duration=0sec 2023-09-06T20:13:52,722 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:52,722 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:52,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031292718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031292722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031292726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031292739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,748 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#C#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:52,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/1b277f6fc4c64ce89219f7901f8cfbe1 is 50, key is test_row_0/C:col10/1694031232561/Put/seqid=0 2023-09-06T20:13:52,749 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/2da321e705fd4cb1af49b1a7c2d07deb is 50, key is test_row_0/C:col10/1694031231051/Put/seqid=0 2023-09-06T20:13:52,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/1b277f6fc4c64ce89219f7901f8cfbe1 2023-09-06T20:13:52,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/d8ababdfea1e4340b03070dbf4e3aaab as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/d8ababdfea1e4340b03070dbf4e3aaab 2023-09-06T20:13:52,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/d8ababdfea1e4340b03070dbf4e3aaab, entries=150, sequenceid=173, filesize=11.8 K 2023-09-06T20:13:52,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/185bbf9fb325422ab5e56a4c9904dfa7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/185bbf9fb325422ab5e56a4c9904dfa7 2023-09-06T20:13:52,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/185bbf9fb325422ab5e56a4c9904dfa7, entries=150, sequenceid=173, filesize=11.8 K 2023-09-06T20:13:52,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/1b277f6fc4c64ce89219f7901f8cfbe1 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/1b277f6fc4c64ce89219f7901f8cfbe1 2023-09-06T20:13:52,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/1b277f6fc4c64ce89219f7901f8cfbe1, entries=150, sequenceid=173, filesize=11.8 K 2023-09-06T20:13:52,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a659f678dfa70a1da8b63a91043e10d5 in 272ms, sequenceid=173, compaction requested=false 2023-09-06T20:13:52,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:52,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2023-09-06T20:13:52,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:52,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:52,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:52,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:52,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:52,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:52,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/2d2aedd8be4a441983339846f91169c0 is 50, key is test_row_0/A:col10/1694031232610/Put/seqid=0 2023-09-06T20:13:52,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031292953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031292955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031292960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031292961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:52,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/2d2aedd8be4a441983339846f91169c0 2023-09-06T20:13:52,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/0013d9daef4d49a29acf38f78f5dd7a2 is 50, key is test_row_0/B:col10/1694031232610/Put/seqid=0 2023-09-06T20:13:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031293062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031293062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031293065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031293065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,217 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/2da321e705fd4cb1af49b1a7c2d07deb as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/2da321e705fd4cb1af49b1a7c2d07deb 2023-09-06T20:13:53,231 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/C of a659f678dfa70a1da8b63a91043e10d5 into 2da321e705fd4cb1af49b1a7c2d07deb(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:53,231 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:53,231 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/C, priority=12, startTime=1694031232557; duration=0sec 2023-09-06T20:13:53,231 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:53,231 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:53,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031293266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031293267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031293269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031293270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/0013d9daef4d49a29acf38f78f5dd7a2 2023-09-06T20:13:53,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/9d1844be2c894bcaa978ec00f765ec58 is 50, key is test_row_0/C:col10/1694031232610/Put/seqid=0 2023-09-06T20:13:53,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/9d1844be2c894bcaa978ec00f765ec58 2023-09-06T20:13:53,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/2d2aedd8be4a441983339846f91169c0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2d2aedd8be4a441983339846f91169c0 2023-09-06T20:13:53,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2d2aedd8be4a441983339846f91169c0, entries=200, sequenceid=195, filesize=14.2 K 2023-09-06T20:13:53,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/0013d9daef4d49a29acf38f78f5dd7a2 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0013d9daef4d49a29acf38f78f5dd7a2 2023-09-06T20:13:53,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0013d9daef4d49a29acf38f78f5dd7a2, entries=150, sequenceid=195, filesize=11.8 K 2023-09-06T20:13:53,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/9d1844be2c894bcaa978ec00f765ec58 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9d1844be2c894bcaa978ec00f765ec58 2023-09-06T20:13:53,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9d1844be2c894bcaa978ec00f765ec58, entries=150, sequenceid=195, filesize=11.8 K 2023-09-06T20:13:53,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for a659f678dfa70a1da8b63a91043e10d5 in 561ms, sequenceid=195, compaction requested=true 2023-09-06T20:13:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:13:53,494 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:13:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:53,494 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:53,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:53,497 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39134 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:53,497 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/A is initiating minor compaction (all files) 2023-09-06T20:13:53,497 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/A in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:53,497 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/06bafdf833464e04bd8a1aebbdceae4b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/d8ababdfea1e4340b03070dbf4e3aaab, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2d2aedd8be4a441983339846f91169c0] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=38.2 K 2023-09-06T20:13:53,498 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:53,498 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/B is initiating minor compaction (all files) 2023-09-06T20:13:53,498 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/B in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:53,498 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/9977f7217d5a46669317b16af6a035b4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/185bbf9fb325422ab5e56a4c9904dfa7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0013d9daef4d49a29acf38f78f5dd7a2] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=35.9 K 2023-09-06T20:13:53,499 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 06bafdf833464e04bd8a1aebbdceae4b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1694031231051 2023-09-06T20:13:53,500 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 9977f7217d5a46669317b16af6a035b4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1694031231051 2023-09-06T20:13:53,500 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 185bbf9fb325422ab5e56a4c9904dfa7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1694031232561 2023-09-06T20:13:53,501 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting d8ababdfea1e4340b03070dbf4e3aaab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1694031232561 2023-09-06T20:13:53,501 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 0013d9daef4d49a29acf38f78f5dd7a2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1694031232606 2023-09-06T20:13:53,502 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2d2aedd8be4a441983339846f91169c0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1694031232606 2023-09-06T20:13:53,519 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#B#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:53,520 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/61d8abe5ccfd4af394a1ed8fb64f0a1a is 50, key is test_row_0/B:col10/1694031232610/Put/seqid=0 2023-09-06T20:13:53,523 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#A#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:53,524 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/73a4f7ebe0a342afb640a5b0bd4e2f95 is 50, key is test_row_0/A:col10/1694031232610/Put/seqid=0 2023-09-06T20:13:53,543 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/61d8abe5ccfd4af394a1ed8fb64f0a1a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/61d8abe5ccfd4af394a1ed8fb64f0a1a 2023-09-06T20:13:53,551 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/B of a659f678dfa70a1da8b63a91043e10d5 into 61d8abe5ccfd4af394a1ed8fb64f0a1a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:53,551 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:53,551 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/B, priority=13, startTime=1694031233494; duration=0sec 2023-09-06T20:13:53,551 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:53,551 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:53,551 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:53,552 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:53,552 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/C is initiating minor compaction (all files) 2023-09-06T20:13:53,552 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/C in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:53,553 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/2da321e705fd4cb1af49b1a7c2d07deb, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/1b277f6fc4c64ce89219f7901f8cfbe1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9d1844be2c894bcaa978ec00f765ec58] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=35.9 K 2023-09-06T20:13:53,553 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 2da321e705fd4cb1af49b1a7c2d07deb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1694031231051 2023-09-06T20:13:53,554 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 1b277f6fc4c64ce89219f7901f8cfbe1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1694031232561 2023-09-06T20:13:53,556 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 9d1844be2c894bcaa978ec00f765ec58, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1694031232606 2023-09-06T20:13:53,567 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#C#compaction#41 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:53,568 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/58a10953dfdf4471bf5ea8163ca820e5 is 50, key is test_row_0/C:col10/1694031232610/Put/seqid=0 2023-09-06T20:13:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:53,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2023-09-06T20:13:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:53,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/dba4da29b33642f2857f86492930ce22 is 50, key is test_row_0/A:col10/1694031232954/Put/seqid=0 2023-09-06T20:13:53,603 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/58a10953dfdf4471bf5ea8163ca820e5 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/58a10953dfdf4471bf5ea8163ca820e5 2023-09-06T20:13:53,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031293607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,613 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/C of a659f678dfa70a1da8b63a91043e10d5 into 58a10953dfdf4471bf5ea8163ca820e5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:53,613 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:53,613 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/C, priority=13, startTime=1694031233494; duration=0sec 2023-09-06T20:13:53,613 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:53,613 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:53,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031293610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031293611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031293615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031293615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/dba4da29b33642f2857f86492930ce22 2023-09-06T20:13:53,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/dc22e4c2e4fd4f318b7ad9f2f5080e7f is 50, key is test_row_0/B:col10/1694031232954/Put/seqid=0 2023-09-06T20:13:53,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/dc22e4c2e4fd4f318b7ad9f2f5080e7f 2023-09-06T20:13:53,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/3174afc262bc408ba16f7552a2fc86c9 is 50, key is test_row_0/C:col10/1694031232954/Put/seqid=0 2023-09-06T20:13:53,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031293715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031293717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031293718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/3174afc262bc408ba16f7552a2fc86c9 2023-09-06T20:13:53,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031293719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031293719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/dba4da29b33642f2857f86492930ce22 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/dba4da29b33642f2857f86492930ce22 2023-09-06T20:13:53,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/dba4da29b33642f2857f86492930ce22, entries=200, sequenceid=215, filesize=14.2 K 2023-09-06T20:13:53,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/dc22e4c2e4fd4f318b7ad9f2f5080e7f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/dc22e4c2e4fd4f318b7ad9f2f5080e7f 2023-09-06T20:13:53,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/dc22e4c2e4fd4f318b7ad9f2f5080e7f, entries=150, sequenceid=215, filesize=11.8 K 2023-09-06T20:13:53,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/3174afc262bc408ba16f7552a2fc86c9 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3174afc262bc408ba16f7552a2fc86c9 2023-09-06T20:13:53,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3174afc262bc408ba16f7552a2fc86c9, entries=150, sequenceid=215, filesize=11.8 K 2023-09-06T20:13:53,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for a659f678dfa70a1da8b63a91043e10d5 in 226ms, sequenceid=215, compaction requested=false 2023-09-06T20:13:53,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:13:53,844 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 16 completed 2023-09-06T20:13:53,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:13:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees 2023-09-06T20:13:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-09-06T20:13:53,851 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:13:53,853 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=18, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:13:53,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:13:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:53,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2023-09-06T20:13:53,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:53,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:53,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/cfc4e7f1e32a453299de83b81e753273 is 50, key is test_row_0/A:col10/1694031233609/Put/seqid=0 2023-09-06T20:13:53,945 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/73a4f7ebe0a342afb640a5b0bd4e2f95 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/73a4f7ebe0a342afb640a5b0bd4e2f95 2023-09-06T20:13:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-09-06T20:13:53,954 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/A of a659f678dfa70a1da8b63a91043e10d5 into 73a4f7ebe0a342afb640a5b0bd4e2f95(size=12.3 K), total size for store is 26.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:53,954 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:53,954 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/A, priority=13, startTime=1694031233494; duration=0sec 2023-09-06T20:13:53,954 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:53,954 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:53,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/cfc4e7f1e32a453299de83b81e753273 2023-09-06T20:13:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031293959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031293959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031293971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031293972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031293972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:53,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/d931243386764028ae822b6e272e3f20 is 50, key is test_row_0/B:col10/1694031233609/Put/seqid=0 2023-09-06T20:13:54,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2023-09-06T20:13:54,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:54,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:54,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:54,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(53): pid=19 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:54,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:54,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031294072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031294072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031294077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031294078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031294083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-09-06T20:13:54,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2023-09-06T20:13:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:54,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(53): pid=19 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:54,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:54,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031294277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031294277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031294281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031294282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031294287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2023-09-06T20:13:54,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:54,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:54,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:54,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(53): pid=19 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:54,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:54,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/d931243386764028ae822b6e272e3f20 2023-09-06T20:13:54,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/a780c939b1494108b7a3b2d3c391558d is 50, key is test_row_0/C:col10/1694031233609/Put/seqid=0 2023-09-06T20:13:54,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/a780c939b1494108b7a3b2d3c391558d 2023-09-06T20:13:54,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/cfc4e7f1e32a453299de83b81e753273 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cfc4e7f1e32a453299de83b81e753273 2023-09-06T20:13:54,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cfc4e7f1e32a453299de83b81e753273, entries=200, sequenceid=236, filesize=14.2 K 2023-09-06T20:13:54,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/d931243386764028ae822b6e272e3f20 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d931243386764028ae822b6e272e3f20 2023-09-06T20:13:54,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d931243386764028ae822b6e272e3f20, entries=150, sequenceid=236, filesize=11.8 K 2023-09-06T20:13:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-09-06T20:13:54,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/a780c939b1494108b7a3b2d3c391558d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a780c939b1494108b7a3b2d3c391558d 2023-09-06T20:13:54,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a780c939b1494108b7a3b2d3c391558d, entries=150, sequenceid=236, filesize=11.8 K 2023-09-06T20:13:54,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=100.63 KB/103050 for a659f678dfa70a1da8b63a91043e10d5 in 556ms, sequenceid=236, compaction requested=true 2023-09-06T20:13:54,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:54,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:13:54,479 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:54,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:54,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:13:54,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:54,479 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:54,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2023-09-06T20:13:54,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:54,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:54,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:54,480 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2023-09-06T20:13:54,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:54,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:54,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:54,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:54,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:54,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:54,487 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:54,487 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/B is initiating minor compaction (all files) 2023-09-06T20:13:54,487 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/B in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:54,487 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/61d8abe5ccfd4af394a1ed8fb64f0a1a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/dc22e4c2e4fd4f318b7ad9f2f5080e7f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d931243386764028ae822b6e272e3f20] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=36.0 K 2023-09-06T20:13:54,488 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41626 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:54,488 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/A is initiating minor compaction (all files) 2023-09-06T20:13:54,489 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/A in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:54,489 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/73a4f7ebe0a342afb640a5b0bd4e2f95, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/dba4da29b33642f2857f86492930ce22, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cfc4e7f1e32a453299de83b81e753273] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=40.7 K 2023-09-06T20:13:54,489 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 61d8abe5ccfd4af394a1ed8fb64f0a1a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1694031232606 2023-09-06T20:13:54,489 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 73a4f7ebe0a342afb640a5b0bd4e2f95, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1694031232606 2023-09-06T20:13:54,490 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting dba4da29b33642f2857f86492930ce22, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1694031232948 2023-09-06T20:13:54,491 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting dc22e4c2e4fd4f318b7ad9f2f5080e7f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1694031232954 2023-09-06T20:13:54,491 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting cfc4e7f1e32a453299de83b81e753273, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1694031233607 2023-09-06T20:13:54,494 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting d931243386764028ae822b6e272e3f20, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1694031233607 2023-09-06T20:13:54,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/d90aab3f12154179bc7890bfd5f5c395 is 50, key is test_row_0/A:col10/1694031233968/Put/seqid=0 2023-09-06T20:13:54,508 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#B#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:54,509 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/803ef76280da4b3fb6fde96a05477566 is 50, key is test_row_0/B:col10/1694031233609/Put/seqid=0 2023-09-06T20:13:54,519 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#A#compaction#50 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:54,519 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/c1370773997b48ee809db433daa58c25 is 50, key is test_row_0/A:col10/1694031233609/Put/seqid=0 2023-09-06T20:13:54,566 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/c1370773997b48ee809db433daa58c25 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c1370773997b48ee809db433daa58c25 2023-09-06T20:13:54,576 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/A of a659f678dfa70a1da8b63a91043e10d5 into c1370773997b48ee809db433daa58c25(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:54,576 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:54,577 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/A, priority=13, startTime=1694031234479; duration=0sec 2023-09-06T20:13:54,577 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:54,577 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:54,577 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:54,578 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:54,579 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/C is initiating minor compaction (all files) 2023-09-06T20:13:54,579 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/C in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:54,579 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/58a10953dfdf4471bf5ea8163ca820e5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3174afc262bc408ba16f7552a2fc86c9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a780c939b1494108b7a3b2d3c391558d] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=36.0 K 2023-09-06T20:13:54,579 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 58a10953dfdf4471bf5ea8163ca820e5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1694031232606 2023-09-06T20:13:54,582 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3174afc262bc408ba16f7552a2fc86c9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1694031232954 2023-09-06T20:13:54,582 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting a780c939b1494108b7a3b2d3c391558d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1694031233607 2023-09-06T20:13:54,592 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#C#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:54,592 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/a5a549cf92154d52bf3828fe31bf8998 is 50, key is test_row_0/C:col10/1694031233609/Put/seqid=0 2023-09-06T20:13:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:54,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:54,618 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/a5a549cf92154d52bf3828fe31bf8998 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a5a549cf92154d52bf3828fe31bf8998 2023-09-06T20:13:54,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031294620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031294620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,666 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/C of a659f678dfa70a1da8b63a91043e10d5 into a5a549cf92154d52bf3828fe31bf8998(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:54,666 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:54,666 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/C, priority=13, startTime=1694031234479; duration=0sec 2023-09-06T20:13:54,666 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:54,666 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:54,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031294667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031294667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031294667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031294767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031294768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031294772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031294772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031294773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,923 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/d90aab3f12154179bc7890bfd5f5c395 2023-09-06T20:13:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/7ae48a1600464248a09d3beff30f5ef5 is 50, key is test_row_0/B:col10/1694031233968/Put/seqid=0 2023-09-06T20:13:54,946 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/803ef76280da4b3fb6fde96a05477566 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/803ef76280da4b3fb6fde96a05477566 2023-09-06T20:13:54,955 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/B of a659f678dfa70a1da8b63a91043e10d5 into 803ef76280da4b3fb6fde96a05477566(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-09-06T20:13:54,955 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:54,955 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/B, priority=13, startTime=1694031234479; duration=0sec 2023-09-06T20:13:54,955 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:54,955 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:54,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031294971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031294972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031294974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031294975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:54,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031294975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031295275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031295275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031295278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031295280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031295281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,364 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/7ae48a1600464248a09d3beff30f5ef5 2023-09-06T20:13:55,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/0c42b50036c1493f983c8b8a390bd533 is 50, key is test_row_0/C:col10/1694031233968/Put/seqid=0 2023-09-06T20:13:55,388 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/0c42b50036c1493f983c8b8a390bd533 2023-09-06T20:13:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/d90aab3f12154179bc7890bfd5f5c395 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/d90aab3f12154179bc7890bfd5f5c395 2023-09-06T20:13:55,403 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/d90aab3f12154179bc7890bfd5f5c395, entries=150, sequenceid=255, filesize=11.8 K 2023-09-06T20:13:55,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/7ae48a1600464248a09d3beff30f5ef5 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/7ae48a1600464248a09d3beff30f5ef5 2023-09-06T20:13:55,413 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/7ae48a1600464248a09d3beff30f5ef5, entries=150, sequenceid=255, filesize=11.8 K 2023-09-06T20:13:55,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/0c42b50036c1493f983c8b8a390bd533 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0c42b50036c1493f983c8b8a390bd533 2023-09-06T20:13:55,422 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0c42b50036c1493f983c8b8a390bd533, entries=150, sequenceid=255, filesize=11.8 K 2023-09-06T20:13:55,423 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(3022): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for a659f678dfa70a1da8b63a91043e10d5 in 943ms, sequenceid=255, compaction requested=false 2023-09-06T20:13:55,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:55,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:55,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=19 2023-09-06T20:13:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=19 2023-09-06T20:13:55,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=19, resume processing ppid=18 2023-09-06T20:13:55,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, ppid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5780 sec 2023-09-06T20:13:55,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees in 1.5890 sec 2023-09-06T20:13:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:55,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2023-09-06T20:13:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:55,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/1792e24106d04db39b74d471a1d4800a is 50, key is test_row_0/A:col10/1694031235782/Put/seqid=0 2023-09-06T20:13:55,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031295815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031295816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031295819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031295820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031295821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/1792e24106d04db39b74d471a1d4800a 2023-09-06T20:13:55,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/70e36316f3c84caab5e5128a4192f915 is 50, key is test_row_0/B:col10/1694031235782/Put/seqid=0 2023-09-06T20:13:55,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031295921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031295921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031295924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031295923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031295935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-09-06T20:13:55,956 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 18 completed 2023-09-06T20:13:55,958 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:13:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees 2023-09-06T20:13:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=20 2023-09-06T20:13:55,965 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:13:55,966 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=20, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:13:55,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:13:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=20 2023-09-06T20:13:56,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2023-09-06T20:13:56,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:56,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031296126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031296126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031296127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031296128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031296144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/70e36316f3c84caab5e5128a4192f915 2023-09-06T20:13:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=20 2023-09-06T20:13:56,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2023-09-06T20:13:56,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:56,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/7cd1309a98e54412b168009c517f96ee is 50, key is test_row_0/C:col10/1694031235782/Put/seqid=0 2023-09-06T20:13:56,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/7cd1309a98e54412b168009c517f96ee 2023-09-06T20:13:56,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/1792e24106d04db39b74d471a1d4800a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/1792e24106d04db39b74d471a1d4800a 2023-09-06T20:13:56,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/1792e24106d04db39b74d471a1d4800a, entries=150, sequenceid=278, filesize=12.0 K 2023-09-06T20:13:56,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/70e36316f3c84caab5e5128a4192f915 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/70e36316f3c84caab5e5128a4192f915 2023-09-06T20:13:56,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/70e36316f3c84caab5e5128a4192f915, entries=150, sequenceid=278, filesize=12.0 K 2023-09-06T20:13:56,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/7cd1309a98e54412b168009c517f96ee as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/7cd1309a98e54412b168009c517f96ee 2023-09-06T20:13:56,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/7cd1309a98e54412b168009c517f96ee, entries=150, sequenceid=278, filesize=12.0 K 2023-09-06T20:13:56,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for a659f678dfa70a1da8b63a91043e10d5 in 545ms, sequenceid=278, compaction requested=true 2023-09-06T20:13:56,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:56,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:13:56,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:56,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:13:56,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:56,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:56,328 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:56,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:56,328 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:56,330 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37098 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:56,330 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/A is initiating minor compaction (all files) 2023-09-06T20:13:56,330 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/A in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,330 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c1370773997b48ee809db433daa58c25, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/d90aab3f12154179bc7890bfd5f5c395, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/1792e24106d04db39b74d471a1d4800a] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=36.2 K 2023-09-06T20:13:56,330 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37098 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:56,331 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/B is initiating minor compaction (all files) 2023-09-06T20:13:56,331 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/B in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,331 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/803ef76280da4b3fb6fde96a05477566, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/7ae48a1600464248a09d3beff30f5ef5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/70e36316f3c84caab5e5128a4192f915] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=36.2 K 2023-09-06T20:13:56,331 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting c1370773997b48ee809db433daa58c25, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1694031233607 2023-09-06T20:13:56,332 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 803ef76280da4b3fb6fde96a05477566, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1694031233607 2023-09-06T20:13:56,332 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting d90aab3f12154179bc7890bfd5f5c395, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1694031233952 2023-09-06T20:13:56,333 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 7ae48a1600464248a09d3beff30f5ef5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1694031233952 2023-09-06T20:13:56,333 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1792e24106d04db39b74d471a1d4800a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1694031234615 2023-09-06T20:13:56,335 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 70e36316f3c84caab5e5128a4192f915, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1694031234615 2023-09-06T20:13:56,361 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#B#compaction#57 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:56,362 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/7001ee4fb13047ab8edec7c2d51c6715 is 50, key is test_row_0/B:col10/1694031235782/Put/seqid=0 2023-09-06T20:13:56,370 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#A#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:56,371 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/cac999c88e5c481e83e82cc7f18724b2 is 50, key is test_row_0/A:col10/1694031235782/Put/seqid=0 2023-09-06T20:13:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:56,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2023-09-06T20:13:56,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:56,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2023-09-06T20:13:56,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:56,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:56,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:56,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:56,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:56,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:56,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/cc115800d3814b4d86e6d75dd4ef09ab is 50, key is test_row_0/A:col10/1694031236429/Put/seqid=0 2023-09-06T20:13:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031296457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031296457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031296460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031296460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031296461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/cc115800d3814b4d86e6d75dd4ef09ab 2023-09-06T20:13:56,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/258de79a5fc8459daedfc0e3d4db9efa is 50, key is test_row_0/B:col10/1694031236429/Put/seqid=0 2023-09-06T20:13:56,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/258de79a5fc8459daedfc0e3d4db9efa 2023-09-06T20:13:56,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/b0f747af2e3c4b07b289c7b5e9682592 is 50, key is test_row_0/C:col10/1694031236429/Put/seqid=0 2023-09-06T20:13:56,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/b0f747af2e3c4b07b289c7b5e9682592 2023-09-06T20:13:56,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/cc115800d3814b4d86e6d75dd4ef09ab as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cc115800d3814b4d86e6d75dd4ef09ab 2023-09-06T20:13:56,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cc115800d3814b4d86e6d75dd4ef09ab, entries=150, sequenceid=296, filesize=12.0 K 2023-09-06T20:13:56,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/258de79a5fc8459daedfc0e3d4db9efa as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/258de79a5fc8459daedfc0e3d4db9efa 2023-09-06T20:13:56,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/258de79a5fc8459daedfc0e3d4db9efa, entries=150, sequenceid=296, filesize=12.0 K 2023-09-06T20:13:56,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/b0f747af2e3c4b07b289c7b5e9682592 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b0f747af2e3c4b07b289c7b5e9682592 2023-09-06T20:13:56,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b0f747af2e3c4b07b289c7b5e9682592, entries=150, sequenceid=296, filesize=12.0 K 2023-09-06T20:13:56,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for a659f678dfa70a1da8b63a91043e10d5 in 134ms, sequenceid=296, compaction requested=true 2023-09-06T20:13:56,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:56,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:56,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-06T20:13:56,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:56,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2023-09-06T20:13:56,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:56,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2023-09-06T20:13:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=20 2023-09-06T20:13:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:56,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2023-09-06T20:13:56,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:56,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:56,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:56,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:56,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:56,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:56,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/a8f5409b1d9c47948718cdb7f165e4f3 is 50, key is test_row_0/A:col10/1694031236458/Put/seqid=0 2023-09-06T20:13:56,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/a8f5409b1d9c47948718cdb7f165e4f3 2023-09-06T20:13:56,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031296582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031296582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031296587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031296589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031296589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2023-09-06T20:13:56,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:56,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/434e2e38a55e4fbe80031b5562f8bc56 is 50, key is test_row_0/B:col10/1694031236458/Put/seqid=0 2023-09-06T20:13:56,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/434e2e38a55e4fbe80031b5562f8bc56 2023-09-06T20:13:56,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/0d72023208bd42dab13e06aebe2f9f7f is 50, key is test_row_0/C:col10/1694031236458/Put/seqid=0 2023-09-06T20:13:56,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/0d72023208bd42dab13e06aebe2f9f7f 2023-09-06T20:13:56,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/a8f5409b1d9c47948718cdb7f165e4f3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/a8f5409b1d9c47948718cdb7f165e4f3 2023-09-06T20:13:56,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/a8f5409b1d9c47948718cdb7f165e4f3, entries=250, sequenceid=316, filesize=16.8 K 2023-09-06T20:13:56,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/434e2e38a55e4fbe80031b5562f8bc56 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/434e2e38a55e4fbe80031b5562f8bc56 2023-09-06T20:13:56,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/434e2e38a55e4fbe80031b5562f8bc56, entries=150, sequenceid=316, filesize=12.0 K 2023-09-06T20:13:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/0d72023208bd42dab13e06aebe2f9f7f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0d72023208bd42dab13e06aebe2f9f7f 2023-09-06T20:13:56,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031296697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031296698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031296699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031296699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0d72023208bd42dab13e06aebe2f9f7f, entries=150, sequenceid=316, filesize=12.0 K 2023-09-06T20:13:56,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a659f678dfa70a1da8b63a91043e10d5 in 137ms, sequenceid=316, compaction requested=true 2023-09-06T20:13:56,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:56,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:56,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2023-09-06T20:13:56,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:56,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=6), splitQueue=0 2023-09-06T20:13:56,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:56,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2023-09-06T20:13:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:56,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2023-09-06T20:13:56,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:56,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:56,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:56,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:56,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:56,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:56,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/1b04f285f2734436835df3d5ecea5d7d is 50, key is test_row_0/A:col10/1694031236705/Put/seqid=0 2023-09-06T20:13:56,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2023-09-06T20:13:56,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:56,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/1b04f285f2734436835df3d5ecea5d7d 2023-09-06T20:13:56,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/79a6adcba06441fca93a771003490e8b is 50, key is test_row_0/B:col10/1694031236705/Put/seqid=0 2023-09-06T20:13:56,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/79a6adcba06441fca93a771003490e8b 2023-09-06T20:13:56,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031296786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/a9f35bb2a4b04786ba4c1258a0c776d0 is 50, key is test_row_0/C:col10/1694031236705/Put/seqid=0 2023-09-06T20:13:56,807 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/7001ee4fb13047ab8edec7c2d51c6715 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/7001ee4fb13047ab8edec7c2d51c6715 2023-09-06T20:13:56,815 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/cac999c88e5c481e83e82cc7f18724b2 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cac999c88e5c481e83e82cc7f18724b2 2023-09-06T20:13:56,818 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/B of a659f678dfa70a1da8b63a91043e10d5 into 7001ee4fb13047ab8edec7c2d51c6715(size=12.6 K), total size for store is 36.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:56,818 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:56,818 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/B, priority=13, startTime=1694031236328; duration=0sec 2023-09-06T20:13:56,819 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2023-09-06T20:13:56,819 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:56,819 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2023-09-06T20:13:56,820 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-06T20:13:56,820 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-06T20:13:56,820 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. because compaction request was cancelled 2023-09-06T20:13:56,820 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:56,820 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2023-09-06T20:13:56,823 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61666 starting at candidate #0 after considering 6 permutations with 6 in ratio 2023-09-06T20:13:56,823 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/C is initiating minor compaction (all files) 2023-09-06T20:13:56,823 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/C in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,823 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a5a549cf92154d52bf3828fe31bf8998, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0c42b50036c1493f983c8b8a390bd533, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/7cd1309a98e54412b168009c517f96ee, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b0f747af2e3c4b07b289c7b5e9682592, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0d72023208bd42dab13e06aebe2f9f7f] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=60.2 K 2023-09-06T20:13:56,824 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting a5a549cf92154d52bf3828fe31bf8998, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1694031233607 2023-09-06T20:13:56,824 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 0c42b50036c1493f983c8b8a390bd533, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1694031233952 2023-09-06T20:13:56,825 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 7cd1309a98e54412b168009c517f96ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1694031234615 2023-09-06T20:13:56,825 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b0f747af2e3c4b07b289c7b5e9682592, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1694031235817 2023-09-06T20:13:56,825 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/A of a659f678dfa70a1da8b63a91043e10d5 into cac999c88e5c481e83e82cc7f18724b2(size=12.6 K), total size for store is 41.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:56,825 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:56,825 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/A, priority=13, startTime=1694031236328; duration=0sec 2023-09-06T20:13:56,826 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2023-09-06T20:13:56,826 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:56,826 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 0d72023208bd42dab13e06aebe2f9f7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1694031236458 2023-09-06T20:13:56,826 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:56,826 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2023-09-06T20:13:56,826 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-06T20:13:56,826 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-06T20:13:56,826 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. because compaction request was cancelled 2023-09-06T20:13:56,826 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:56,826 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:56,826 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:56,826 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:56,827 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42380 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:56,828 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/A is initiating minor compaction (all files) 2023-09-06T20:13:56,828 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/A in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,828 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cac999c88e5c481e83e82cc7f18724b2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cc115800d3814b4d86e6d75dd4ef09ab, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/a8f5409b1d9c47948718cdb7f165e4f3] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=41.4 K 2023-09-06T20:13:56,828 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting cac999c88e5c481e83e82cc7f18724b2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1694031234615 2023-09-06T20:13:56,829 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting cc115800d3814b4d86e6d75dd4ef09ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1694031235817 2023-09-06T20:13:56,829 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting a8f5409b1d9c47948718cdb7f165e4f3, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1694031236456 2023-09-06T20:13:56,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/a9f35bb2a4b04786ba4c1258a0c776d0 2023-09-06T20:13:56,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/1b04f285f2734436835df3d5ecea5d7d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/1b04f285f2734436835df3d5ecea5d7d 2023-09-06T20:13:56,838 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#A#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:56,838 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/ee80b7a5350c4ab6a07916ad7e14c7e5 is 50, key is test_row_0/A:col10/1694031236458/Put/seqid=0 2023-09-06T20:13:56,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/1b04f285f2734436835df3d5ecea5d7d, entries=150, sequenceid=333, filesize=12.0 K 2023-09-06T20:13:56,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/79a6adcba06441fca93a771003490e8b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/79a6adcba06441fca93a771003490e8b 2023-09-06T20:13:56,848 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:56,849 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/38c540a71ee74bd0a9c5304aeab82d47 is 50, key is test_row_0/C:col10/1694031236458/Put/seqid=0 2023-09-06T20:13:56,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/79a6adcba06441fca93a771003490e8b, entries=150, sequenceid=333, filesize=12.0 K 2023-09-06T20:13:56,859 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/ee80b7a5350c4ab6a07916ad7e14c7e5 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ee80b7a5350c4ab6a07916ad7e14c7e5 2023-09-06T20:13:56,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/a9f35bb2a4b04786ba4c1258a0c776d0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a9f35bb2a4b04786ba4c1258a0c776d0 2023-09-06T20:13:56,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a9f35bb2a4b04786ba4c1258a0c776d0, entries=150, sequenceid=333, filesize=12.0 K 2023-09-06T20:13:56,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a659f678dfa70a1da8b63a91043e10d5 in 160ms, sequenceid=333, compaction requested=true 2023-09-06T20:13:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:13:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2023-09-06T20:13:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:13:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2023-09-06T20:13:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=6), splitQueue=0 2023-09-06T20:13:56,871 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/38c540a71ee74bd0a9c5304aeab82d47 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/38c540a71ee74bd0a9c5304aeab82d47 2023-09-06T20:13:56,874 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/A of a659f678dfa70a1da8b63a91043e10d5 into ee80b7a5350c4ab6a07916ad7e14c7e5(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:56,875 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:56,875 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/A, priority=13, startTime=1694031236566; duration=0sec 2023-09-06T20:13:56,875 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=6), splitQueue=0 2023-09-06T20:13:56,875 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:56,875 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 5 compacting, 1 eligible, 16 blocking 2023-09-06T20:13:56,878 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-06T20:13:56,878 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-06T20:13:56,878 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. because compaction request was cancelled 2023-09-06T20:13:56,878 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:56,878 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:13:56,880 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:13:56,880 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/B is initiating minor compaction (all files) 2023-09-06T20:13:56,880 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/B in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,880 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/7001ee4fb13047ab8edec7c2d51c6715, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/258de79a5fc8459daedfc0e3d4db9efa, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/434e2e38a55e4fbe80031b5562f8bc56, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/79a6adcba06441fca93a771003490e8b] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=48.6 K 2023-09-06T20:13:56,881 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7001ee4fb13047ab8edec7c2d51c6715, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1694031234615 2023-09-06T20:13:56,881 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 258de79a5fc8459daedfc0e3d4db9efa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1694031235817 2023-09-06T20:13:56,881 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 5 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/C of a659f678dfa70a1da8b63a91043e10d5 into 38c540a71ee74bd0a9c5304aeab82d47(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:56,882 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/C, priority=11, startTime=1694031236706; duration=0sec 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 434e2e38a55e4fbe80031b5562f8bc56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1694031236458 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. because compaction request was cancelled 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. because compaction request was cancelled 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 79a6adcba06441fca93a771003490e8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1694031236580 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:56,882 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2023-09-06T20:13:56,884 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-06T20:13:56,884 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-06T20:13:56,884 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. because compaction request was cancelled 2023-09-06T20:13:56,884 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:56,884 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2023-09-06T20:13:56,886 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-06T20:13:56,886 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-06T20:13:56,886 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. because compaction request was cancelled 2023-09-06T20:13:56,886 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:56,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2023-09-06T20:13:56,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:56,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:56,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:56,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:56,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:56,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:56,900 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#B#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:56,900 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/a5da57ff07e1458cae5f3fe546f3483c is 50, key is test_row_0/B:col10/1694031236705/Put/seqid=0 2023-09-06T20:13:56,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/467d6eb805f04b829653e5a55b81dda3 is 50, key is test_row_0/A:col10/1694031236772/Put/seqid=0 2023-09-06T20:13:56,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2023-09-06T20:13:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:56,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:56,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031296915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031296917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031296919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031296920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031296920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:56,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/467d6eb805f04b829653e5a55b81dda3 2023-09-06T20:13:56,951 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/a5da57ff07e1458cae5f3fe546f3483c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/a5da57ff07e1458cae5f3fe546f3483c 2023-09-06T20:13:56,961 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/B of a659f678dfa70a1da8b63a91043e10d5 into a5da57ff07e1458cae5f3fe546f3483c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:56,961 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:56,961 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/B, priority=12, startTime=1694031236867; duration=0sec 2023-09-06T20:13:56,961 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:56,961 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:56,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/56a40ab8ec4447269501ce3964553bb6 is 50, key is test_row_0/B:col10/1694031236772/Put/seqid=0 2023-09-06T20:13:56,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/56a40ab8ec4447269501ce3964553bb6 2023-09-06T20:13:57,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/9e229d3dcf7d43028cab50e5af4613b3 is 50, key is test_row_0/C:col10/1694031236772/Put/seqid=0 2023-09-06T20:13:57,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031297021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031297022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031297023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/9e229d3dcf7d43028cab50e5af4613b3 2023-09-06T20:13:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031297024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031297025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/467d6eb805f04b829653e5a55b81dda3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/467d6eb805f04b829653e5a55b81dda3 2023-09-06T20:13:57,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/467d6eb805f04b829653e5a55b81dda3, entries=150, sequenceid=357, filesize=12.0 K 2023-09-06T20:13:57,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/56a40ab8ec4447269501ce3964553bb6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/56a40ab8ec4447269501ce3964553bb6 2023-09-06T20:13:57,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/56a40ab8ec4447269501ce3964553bb6, entries=150, sequenceid=357, filesize=12.0 K 2023-09-06T20:13:57,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/9e229d3dcf7d43028cab50e5af4613b3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9e229d3dcf7d43028cab50e5af4613b3 2023-09-06T20:13:57,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9e229d3dcf7d43028cab50e5af4613b3, entries=150, sequenceid=357, filesize=12.0 K 2023-09-06T20:13:57,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a659f678dfa70a1da8b63a91043e10d5 in 159ms, sequenceid=357, compaction requested=true 2023-09-06T20:13:57,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:57,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:13:57,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:57,056 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:57,056 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2023-09-06T20:13:57,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:13:57,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:57,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:57,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:57,057 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-06T20:13:57,057 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-06T20:13:57,057 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. because compaction request was cancelled 2023-09-06T20:13:57,057 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:57,057 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37602 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:57,057 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/A is initiating minor compaction (all files) 2023-09-06T20:13:57,057 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:57,057 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/A in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:57,057 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ee80b7a5350c4ab6a07916ad7e14c7e5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/1b04f285f2734436835df3d5ecea5d7d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/467d6eb805f04b829653e5a55b81dda3] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=36.7 K 2023-09-06T20:13:57,058 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting ee80b7a5350c4ab6a07916ad7e14c7e5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1694031236458 2023-09-06T20:13:57,058 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 1b04f285f2734436835df3d5ecea5d7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1694031236580 2023-09-06T20:13:57,059 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 467d6eb805f04b829653e5a55b81dda3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1694031236772 2023-09-06T20:13:57,059 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37568 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:57,059 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/C is initiating minor compaction (all files) 2023-09-06T20:13:57,059 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/C in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:57,059 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/38c540a71ee74bd0a9c5304aeab82d47, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a9f35bb2a4b04786ba4c1258a0c776d0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9e229d3dcf7d43028cab50e5af4613b3] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=36.7 K 2023-09-06T20:13:57,060 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 38c540a71ee74bd0a9c5304aeab82d47, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1694031236458 2023-09-06T20:13:57,061 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting a9f35bb2a4b04786ba4c1258a0c776d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1694031236580 2023-09-06T20:13:57,061 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9e229d3dcf7d43028cab50e5af4613b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1694031236772 2023-09-06T20:13:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=20 2023-09-06T20:13:57,071 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#A#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:57,071 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/695f38e7890840809455ac65df5f56fa is 50, key is test_row_0/A:col10/1694031236772/Put/seqid=0 2023-09-06T20:13:57,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2023-09-06T20:13:57,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:57,078 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2023-09-06T20:13:57,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:57,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:57,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:57,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:57,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:57,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:57,087 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#C#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/b2eb83c4e9724754af6537f7af0f0fe6 is 50, key is test_row_0/A:col10/1694031236918/Put/seqid=0 2023-09-06T20:13:57,088 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/c150c1e4e7be465d8c75d9f9c8dc6085 is 50, key is test_row_0/C:col10/1694031236772/Put/seqid=0 2023-09-06T20:13:57,110 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/b2eb83c4e9724754af6537f7af0f0fe6 2023-09-06T20:13:57,118 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/c150c1e4e7be465d8c75d9f9c8dc6085 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/c150c1e4e7be465d8c75d9f9c8dc6085 2023-09-06T20:13:57,128 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/C of a659f678dfa70a1da8b63a91043e10d5 into c150c1e4e7be465d8c75d9f9c8dc6085(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:57,128 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:57,128 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/C, priority=13, startTime=1694031237056; duration=0sec 2023-09-06T20:13:57,128 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:57,128 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:57,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/b915b6978c1b40fcbb385ca3baae4cc4 is 50, key is test_row_0/B:col10/1694031236918/Put/seqid=0 2023-09-06T20:13:57,149 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/b915b6978c1b40fcbb385ca3baae4cc4 2023-09-06T20:13:57,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/4ca2359f34fa4b9baab4f4d8c9aec792 is 50, key is test_row_0/C:col10/1694031236918/Put/seqid=0 2023-09-06T20:13:57,173 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/4ca2359f34fa4b9baab4f4d8c9aec792 2023-09-06T20:13:57,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/b2eb83c4e9724754af6537f7af0f0fe6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/b2eb83c4e9724754af6537f7af0f0fe6 2023-09-06T20:13:57,188 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/b2eb83c4e9724754af6537f7af0f0fe6, entries=150, sequenceid=374, filesize=12.0 K 2023-09-06T20:13:57,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/b915b6978c1b40fcbb385ca3baae4cc4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b915b6978c1b40fcbb385ca3baae4cc4 2023-09-06T20:13:57,196 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b915b6978c1b40fcbb385ca3baae4cc4, entries=150, sequenceid=374, filesize=12.0 K 2023-09-06T20:13:57,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/4ca2359f34fa4b9baab4f4d8c9aec792 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/4ca2359f34fa4b9baab4f4d8c9aec792 2023-09-06T20:13:57,214 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/4ca2359f34fa4b9baab4f4d8c9aec792, entries=150, sequenceid=374, filesize=12.0 K 2023-09-06T20:13:57,215 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for a659f678dfa70a1da8b63a91043e10d5 in 137ms, sequenceid=374, compaction requested=true 2023-09-06T20:13:57,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:57,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:57,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=21 2023-09-06T20:13:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=21 2023-09-06T20:13:57,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-09-06T20:13:57,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2520 sec 2023-09-06T20:13:57,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees in 1.2650 sec 2023-09-06T20:13:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:57,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:13:57,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:57,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:57,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:57,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:57,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:57,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:57,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/aab3ca2517cf4d1ba1bcbdc823295073 is 50, key is test_row_0/A:col10/1694031237235/Put/seqid=0 2023-09-06T20:13:57,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/aab3ca2517cf4d1ba1bcbdc823295073 2023-09-06T20:13:57,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/214e3692091040658d21c57f4ac164f8 is 50, key is test_row_0/B:col10/1694031237235/Put/seqid=0 2023-09-06T20:13:57,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031297296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031297303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031297306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031297307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031297314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/214e3692091040658d21c57f4ac164f8 2023-09-06T20:13:57,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/d599fa6b6d434d128121cd68110c73de is 50, key is test_row_0/C:col10/1694031237235/Put/seqid=0 2023-09-06T20:13:57,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/d599fa6b6d434d128121cd68110c73de 2023-09-06T20:13:57,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/aab3ca2517cf4d1ba1bcbdc823295073 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/aab3ca2517cf4d1ba1bcbdc823295073 2023-09-06T20:13:57,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/aab3ca2517cf4d1ba1bcbdc823295073, entries=300, sequenceid=386, filesize=19.1 K 2023-09-06T20:13:57,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/214e3692091040658d21c57f4ac164f8 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/214e3692091040658d21c57f4ac164f8 2023-09-06T20:13:57,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/214e3692091040658d21c57f4ac164f8, entries=150, sequenceid=386, filesize=12.0 K 2023-09-06T20:13:57,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/d599fa6b6d434d128121cd68110c73de as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d599fa6b6d434d128121cd68110c73de 2023-09-06T20:13:57,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d599fa6b6d434d128121cd68110c73de, entries=150, sequenceid=386, filesize=12.0 K 2023-09-06T20:13:57,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a659f678dfa70a1da8b63a91043e10d5 in 163ms, sequenceid=386, compaction requested=true 2023-09-06T20:13:57,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:57,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:13:57,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:57,402 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2023-09-06T20:13:57,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:13:57,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:57,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:57,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-06T20:13:57,403 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-06T20:13:57,403 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-06T20:13:57,403 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. because compaction request was cancelled 2023-09-06T20:13:57,403 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:57,403 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:13:57,405 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:13:57,405 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/B is initiating minor compaction (all files) 2023-09-06T20:13:57,405 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/B in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:57,405 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/a5da57ff07e1458cae5f3fe546f3483c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/56a40ab8ec4447269501ce3964553bb6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b915b6978c1b40fcbb385ca3baae4cc4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/214e3692091040658d21c57f4ac164f8] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=48.8 K 2023-09-06T20:13:57,406 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting a5da57ff07e1458cae5f3fe546f3483c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1694031236580 2023-09-06T20:13:57,406 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 56a40ab8ec4447269501ce3964553bb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1694031236772 2023-09-06T20:13:57,407 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting b915b6978c1b40fcbb385ca3baae4cc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1694031236917 2023-09-06T20:13:57,407 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 214e3692091040658d21c57f4ac164f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1694031237235 2023-09-06T20:13:57,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-06T20:13:57,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:57,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:57,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:57,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:57,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:57,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:57,427 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#B#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:57,428 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/e545c21e23e8444f805867afb93f1da2 is 50, key is test_row_0/B:col10/1694031237235/Put/seqid=0 2023-09-06T20:13:57,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031297426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031297427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031297428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031297430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031297431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/84b71622ebaf4c9bac9a1a749ea1376a is 50, key is test_row_0/A:col10/1694031237305/Put/seqid=0 2023-09-06T20:13:57,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/84b71622ebaf4c9bac9a1a749ea1376a 2023-09-06T20:13:57,473 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/e545c21e23e8444f805867afb93f1da2 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/e545c21e23e8444f805867afb93f1da2 2023-09-06T20:13:57,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/375259bb852344908484384a0a2a4888 is 50, key is test_row_0/B:col10/1694031237305/Put/seqid=0 2023-09-06T20:13:57,482 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/B of a659f678dfa70a1da8b63a91043e10d5 into e545c21e23e8444f805867afb93f1da2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:57,482 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:57,483 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/B, priority=12, startTime=1694031237402; duration=0sec 2023-09-06T20:13:57,483 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:57,483 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:57,483 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:57,484 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37670 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:57,484 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/C is initiating minor compaction (all files) 2023-09-06T20:13:57,484 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/C in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:57,484 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/c150c1e4e7be465d8c75d9f9c8dc6085, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/4ca2359f34fa4b9baab4f4d8c9aec792, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d599fa6b6d434d128121cd68110c73de] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=36.8 K 2023-09-06T20:13:57,485 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting c150c1e4e7be465d8c75d9f9c8dc6085, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1694031236772 2023-09-06T20:13:57,485 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4ca2359f34fa4b9baab4f4d8c9aec792, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1694031236917 2023-09-06T20:13:57,486 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting d599fa6b6d434d128121cd68110c73de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1694031237235 2023-09-06T20:13:57,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/375259bb852344908484384a0a2a4888 2023-09-06T20:13:57,494 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#C#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:57,495 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/6ff1ba35251046d883c7b3bb8961f2eb is 50, key is test_row_0/C:col10/1694031237235/Put/seqid=0 2023-09-06T20:13:57,501 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/695f38e7890840809455ac65df5f56fa as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/695f38e7890840809455ac65df5f56fa 2023-09-06T20:13:57,508 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/A of a659f678dfa70a1da8b63a91043e10d5 into 695f38e7890840809455ac65df5f56fa(size=12.8 K), total size for store is 44.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:57,508 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:57,508 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/A, priority=13, startTime=1694031237055; duration=0sec 2023-09-06T20:13:57,508 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:57,508 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:57,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/3e5dfc01a0fe45e4bc3bdac775054482 is 50, key is test_row_0/C:col10/1694031237305/Put/seqid=0 2023-09-06T20:13:57,525 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/6ff1ba35251046d883c7b3bb8961f2eb as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6ff1ba35251046d883c7b3bb8961f2eb 2023-09-06T20:13:57,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031297533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031297534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031297534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031297536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031297536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,538 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/C of a659f678dfa70a1da8b63a91043e10d5 into 6ff1ba35251046d883c7b3bb8961f2eb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:57,538 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:57,538 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/C, priority=13, startTime=1694031237402; duration=0sec 2023-09-06T20:13:57,538 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:57,539 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:57,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031297737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031297738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031297738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031297739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031297739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:57,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/3e5dfc01a0fe45e4bc3bdac775054482 2023-09-06T20:13:57,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/84b71622ebaf4c9bac9a1a749ea1376a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/84b71622ebaf4c9bac9a1a749ea1376a 2023-09-06T20:13:57,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/84b71622ebaf4c9bac9a1a749ea1376a, entries=200, sequenceid=412, filesize=14.4 K 2023-09-06T20:13:57,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/375259bb852344908484384a0a2a4888 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/375259bb852344908484384a0a2a4888 2023-09-06T20:13:57,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/375259bb852344908484384a0a2a4888, entries=150, sequenceid=412, filesize=12.0 K 2023-09-06T20:13:57,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/3e5dfc01a0fe45e4bc3bdac775054482 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3e5dfc01a0fe45e4bc3bdac775054482 2023-09-06T20:13:57,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3e5dfc01a0fe45e4bc3bdac775054482, entries=150, sequenceid=412, filesize=12.0 K 2023-09-06T20:13:57,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a659f678dfa70a1da8b63a91043e10d5 in 551ms, sequenceid=412, compaction requested=true 2023-09-06T20:13:57,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:57,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:13:57,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:57,969 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:13:57,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:13:57,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:57,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:57,969 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2023-09-06T20:13:57,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:57,970 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-06T20:13:57,971 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-06T20:13:57,971 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. because compaction request was cancelled 2023-09-06T20:13:57,971 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:57,971 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2023-09-06T20:13:57,972 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59748 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:13:57,972 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/A is initiating minor compaction (all files) 2023-09-06T20:13:57,972 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-06T20:13:57,972 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/A in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:57,972 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-06T20:13:57,972 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. because compaction request was cancelled 2023-09-06T20:13:57,972 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:57,972 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/695f38e7890840809455ac65df5f56fa, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/b2eb83c4e9724754af6537f7af0f0fe6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/aab3ca2517cf4d1ba1bcbdc823295073, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/84b71622ebaf4c9bac9a1a749ea1376a] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=58.3 K 2023-09-06T20:13:57,974 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 695f38e7890840809455ac65df5f56fa, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1694031236772 2023-09-06T20:13:57,974 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting b2eb83c4e9724754af6537f7af0f0fe6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1694031236917 2023-09-06T20:13:57,975 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting aab3ca2517cf4d1ba1bcbdc823295073, keycount=300, bloomtype=ROW, size=19.1 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1694031237232 2023-09-06T20:13:57,979 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 84b71622ebaf4c9bac9a1a749ea1376a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1694031237301 2023-09-06T20:13:57,991 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:57,992 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/b08f4500565342f18fdd9c2c483631ca is 50, key is test_row_0/A:col10/1694031237305/Put/seqid=0 2023-09-06T20:13:58,029 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/b08f4500565342f18fdd9c2c483631ca as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/b08f4500565342f18fdd9c2c483631ca 2023-09-06T20:13:58,038 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/A of a659f678dfa70a1da8b63a91043e10d5 into b08f4500565342f18fdd9c2c483631ca(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:58,038 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:58,038 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/A, priority=12, startTime=1694031237969; duration=0sec 2023-09-06T20:13:58,039 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:58,039 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:58,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:13:58,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:58,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:58,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:58,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:58,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:58,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:58,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/245a742276e14a158efcd33dd9284ac6 is 50, key is test_row_0/A:col10/1694031238044/Put/seqid=0 2023-09-06T20:13:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=20 2023-09-06T20:13:58,071 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 20 completed 2023-09-06T20:13:58,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:13:58,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees 2023-09-06T20:13:58,076 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=22, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:13:58,077 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=22, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:13:58,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:13:58,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:13:58,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031298070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031298072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031298074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031298080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031298073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/245a742276e14a158efcd33dd9284ac6 2023-09-06T20:13:58,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/fdd0a209f2984bc3850a7f80ca299e8e is 50, key is test_row_0/B:col10/1694031238044/Put/seqid=0 2023-09-06T20:13:58,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/fdd0a209f2984bc3850a7f80ca299e8e 2023-09-06T20:13:58,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/b8ed8a45e03d4196b430622a85c6bcbf is 50, key is test_row_0/C:col10/1694031238044/Put/seqid=0 2023-09-06T20:13:58,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/b8ed8a45e03d4196b430622a85c6bcbf 2023-09-06T20:13:58,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/245a742276e14a158efcd33dd9284ac6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/245a742276e14a158efcd33dd9284ac6 2023-09-06T20:13:58,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/245a742276e14a158efcd33dd9284ac6, entries=150, sequenceid=431, filesize=12.0 K 2023-09-06T20:13:58,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/fdd0a209f2984bc3850a7f80ca299e8e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/fdd0a209f2984bc3850a7f80ca299e8e 2023-09-06T20:13:58,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/fdd0a209f2984bc3850a7f80ca299e8e, entries=150, sequenceid=431, filesize=12.0 K 2023-09-06T20:13:58,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/b8ed8a45e03d4196b430622a85c6bcbf as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b8ed8a45e03d4196b430622a85c6bcbf 2023-09-06T20:13:58,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b8ed8a45e03d4196b430622a85c6bcbf, entries=150, sequenceid=431, filesize=12.0 K 2023-09-06T20:13:58,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=134.18 KB/137400 for a659f678dfa70a1da8b63a91043e10d5 in 132ms, sequenceid=431, compaction requested=true 2023-09-06T20:13:58,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:58,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:13:58,178 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2023-09-06T20:13:58,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:58,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:13:58,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:58,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:58,179 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:58,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:58,180 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-06T20:13:58,180 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-06T20:13:58,180 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. because compaction request was cancelled 2023-09-06T20:13:58,180 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:58,180 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:58,181 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37772 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:58,181 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/B is initiating minor compaction (all files) 2023-09-06T20:13:58,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:13:58,181 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/B in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:58,181 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/e545c21e23e8444f805867afb93f1da2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/375259bb852344908484384a0a2a4888, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/fdd0a209f2984bc3850a7f80ca299e8e] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=36.9 K 2023-09-06T20:13:58,182 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting e545c21e23e8444f805867afb93f1da2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1694031237235 2023-09-06T20:13:58,182 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37772 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:58,182 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/C is initiating minor compaction (all files) 2023-09-06T20:13:58,182 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/C in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:58,182 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6ff1ba35251046d883c7b3bb8961f2eb, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3e5dfc01a0fe45e4bc3bdac775054482, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b8ed8a45e03d4196b430622a85c6bcbf] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=36.9 K 2023-09-06T20:13:58,183 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 375259bb852344908484384a0a2a4888, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1694031237305 2023-09-06T20:13:58,183 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6ff1ba35251046d883c7b3bb8961f2eb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1694031237235 2023-09-06T20:13:58,183 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting fdd0a209f2984bc3850a7f80ca299e8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1694031238044 2023-09-06T20:13:58,184 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3e5dfc01a0fe45e4bc3bdac775054482, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1694031237305 2023-09-06T20:13:58,184 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting b8ed8a45e03d4196b430622a85c6bcbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1694031238044 2023-09-06T20:13:58,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:58,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-06T20:13:58,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:58,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:58,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:58,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:58,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:58,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:58,198 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:58,199 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/83cc07450b994b7a87642390901b9897 is 50, key is test_row_0/B:col10/1694031238044/Put/seqid=0 2023-09-06T20:13:58,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031298200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031298200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031298201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031298201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031298201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,207 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:58,208 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/310a78ec80fd4f139b75f98a9ef6cfdb is 50, key is test_row_0/C:col10/1694031238044/Put/seqid=0 2023-09-06T20:13:58,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/2471e1d6348b460cb9f7ae300e144ff7 is 50, key is test_row_0/A:col10/1694031238192/Put/seqid=0 2023-09-06T20:13:58,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2023-09-06T20:13:58,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:58,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:58,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:58,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(53): pid=23 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:58,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:13:58,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/2471e1d6348b460cb9f7ae300e144ff7 2023-09-06T20:13:58,244 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/83cc07450b994b7a87642390901b9897 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/83cc07450b994b7a87642390901b9897 2023-09-06T20:13:58,251 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/B of a659f678dfa70a1da8b63a91043e10d5 into 83cc07450b994b7a87642390901b9897(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:58,251 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:58,251 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/B, priority=13, startTime=1694031238178; duration=0sec 2023-09-06T20:13:58,252 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:58,252 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:58,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/4045171879314bdb870b75bc9205333b is 50, key is test_row_0/B:col10/1694031238192/Put/seqid=0 2023-09-06T20:13:58,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/4045171879314bdb870b75bc9205333b 2023-09-06T20:13:58,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/b6a7c58e47c44224b222c5547ac431df is 50, key is test_row_0/C:col10/1694031238192/Put/seqid=0 2023-09-06T20:13:58,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/b6a7c58e47c44224b222c5547ac431df 2023-09-06T20:13:58,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031298305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031298306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031298306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031298306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031298308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/2471e1d6348b460cb9f7ae300e144ff7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2471e1d6348b460cb9f7ae300e144ff7 2023-09-06T20:13:58,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2471e1d6348b460cb9f7ae300e144ff7, entries=250, sequenceid=459, filesize=16.8 K 2023-09-06T20:13:58,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/4045171879314bdb870b75bc9205333b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/4045171879314bdb870b75bc9205333b 2023-09-06T20:13:58,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/4045171879314bdb870b75bc9205333b, entries=150, sequenceid=459, filesize=12.0 K 2023-09-06T20:13:58,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/b6a7c58e47c44224b222c5547ac431df as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b6a7c58e47c44224b222c5547ac431df 2023-09-06T20:13:58,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b6a7c58e47c44224b222c5547ac431df, entries=150, sequenceid=459, filesize=12.0 K 2023-09-06T20:13:58,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for a659f678dfa70a1da8b63a91043e10d5 in 144ms, sequenceid=459, compaction requested=true 2023-09-06T20:13:58,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:58,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:13:58,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:58,336 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:13:58,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:58,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:13:58,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:13:58,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-06T20:13:58,338 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42720 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:13:58,338 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/A is initiating minor compaction (all files) 2023-09-06T20:13:58,338 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/A in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:58,338 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/b08f4500565342f18fdd9c2c483631ca, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/245a742276e14a158efcd33dd9284ac6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2471e1d6348b460cb9f7ae300e144ff7] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=41.7 K 2023-09-06T20:13:58,339 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b08f4500565342f18fdd9c2c483631ca, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1694031237305 2023-09-06T20:13:58,339 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 245a742276e14a158efcd33dd9284ac6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1694031238044 2023-09-06T20:13:58,342 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 2471e1d6348b460cb9f7ae300e144ff7, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1694031238072 2023-09-06T20:13:58,352 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:13:58,352 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/fa89ed69f19c4d50bf0b3b81d246b993 is 50, key is test_row_0/A:col10/1694031238192/Put/seqid=0 2023-09-06T20:13:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:13:58,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2023-09-06T20:13:58,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:58,394 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2023-09-06T20:13:58,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:58,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:58,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:58,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:58,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:58,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:58,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/5b5edbf252d745ab9dd571c676aee69a is 50, key is test_row_1/A:col10/1694031238200/Put/seqid=0 2023-09-06T20:13:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:58,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:13:58,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031298539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031298540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031298543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031298541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031298544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,634 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/310a78ec80fd4f139b75f98a9ef6cfdb as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/310a78ec80fd4f139b75f98a9ef6cfdb 2023-09-06T20:13:58,645 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/C of a659f678dfa70a1da8b63a91043e10d5 into 310a78ec80fd4f139b75f98a9ef6cfdb(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:58,645 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:58,645 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/C, priority=13, startTime=1694031238179; duration=0sec 2023-09-06T20:13:58,645 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-06T20:13:58,645 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:58,645 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:58,646 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2023-09-06T20:13:58,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031298646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,648 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-06T20:13:58,648 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-06T20:13:58,648 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. because compaction request was cancelled 2023-09-06T20:13:58,648 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:13:58,648 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2023-09-06T20:13:58,649 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-06T20:13:58,649 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-06T20:13:58,649 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. because compaction request was cancelled 2023-09-06T20:13:58,649 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:13:58,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031298647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031298647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031298647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031298648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:13:58,766 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/fa89ed69f19c4d50bf0b3b81d246b993 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/fa89ed69f19c4d50bf0b3b81d246b993 2023-09-06T20:13:58,773 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/A of a659f678dfa70a1da8b63a91043e10d5 into fa89ed69f19c4d50bf0b3b81d246b993(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:13:58,773 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:58,773 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/A, priority=13, startTime=1694031238336; duration=0sec 2023-09-06T20:13:58,773 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:13:58,774 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:13:58,818 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/5b5edbf252d745ab9dd571c676aee69a 2023-09-06T20:13:58,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/d0fce4b1be0f4239a3d652b4128541a6 is 50, key is test_row_1/B:col10/1694031238200/Put/seqid=0 2023-09-06T20:13:58,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031298849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031298852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031298852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031298852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:58,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031298854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031299152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031299155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031299156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031299156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031299157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:13:59,241 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/d0fce4b1be0f4239a3d652b4128541a6 2023-09-06T20:13:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/28760f2b3e5b46f1bbfa2b3272e7683f is 50, key is test_row_1/C:col10/1694031238200/Put/seqid=0 2023-09-06T20:13:59,272 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/28760f2b3e5b46f1bbfa2b3272e7683f 2023-09-06T20:13:59,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/5b5edbf252d745ab9dd571c676aee69a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/5b5edbf252d745ab9dd571c676aee69a 2023-09-06T20:13:59,293 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/5b5edbf252d745ab9dd571c676aee69a, entries=100, sequenceid=468, filesize=9.6 K 2023-09-06T20:13:59,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/d0fce4b1be0f4239a3d652b4128541a6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d0fce4b1be0f4239a3d652b4128541a6 2023-09-06T20:13:59,305 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d0fce4b1be0f4239a3d652b4128541a6, entries=100, sequenceid=468, filesize=9.6 K 2023-09-06T20:13:59,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/28760f2b3e5b46f1bbfa2b3272e7683f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/28760f2b3e5b46f1bbfa2b3272e7683f 2023-09-06T20:13:59,312 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/28760f2b3e5b46f1bbfa2b3272e7683f, entries=100, sequenceid=468, filesize=9.6 K 2023-09-06T20:13:59,313 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(3022): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for a659f678dfa70a1da8b63a91043e10d5 in 919ms, sequenceid=468, compaction requested=true 2023-09-06T20:13:59,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:13:59,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:13:59,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=23 2023-09-06T20:13:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=23 2023-09-06T20:13:59,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-09-06T20:13:59,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2390 sec 2023-09-06T20:13:59,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees in 1.2450 sec 2023-09-06T20:13:59,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:13:59,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2023-09-06T20:13:59,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:13:59,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:59,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:13:59,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:59,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:13:59,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:13:59,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/7388978ff3854364b2f5ef1a8b41b62f is 50, key is test_row_0/A:col10/1694031238542/Put/seqid=0 2023-09-06T20:13:59,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031299668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031299664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031299671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031299671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031299673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/7388978ff3854364b2f5ef1a8b41b62f 2023-09-06T20:13:59,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/f990f5bf9f6d419ba3aca9aefb8eabfb is 50, key is test_row_0/B:col10/1694031238542/Put/seqid=0 2023-09-06T20:13:59,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031299772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031299773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031299775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031299776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031299775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031299974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031299976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031299977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031299978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:13:59,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:13:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031299980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/f990f5bf9f6d419ba3aca9aefb8eabfb 2023-09-06T20:14:00,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/6ef21b673c3545ee96acb63cc5aa2bbc is 50, key is test_row_0/C:col10/1694031238542/Put/seqid=0 2023-09-06T20:14:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:14:00,187 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 22 completed 2023-09-06T20:14:00,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=24, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees 2023-09-06T20:14:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-09-06T20:14:00,192 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=24, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:00,192 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=24, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:00,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:00,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031300276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031300278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031300280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031300282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031300284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-09-06T20:14:00,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2023-09-06T20:14:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:14:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:00,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] handler.RSProcedureHandler(53): pid=25 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=25 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=25 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-09-06T20:14:00,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2023-09-06T20:14:00,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:00,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:14:00,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:00,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] handler.RSProcedureHandler(53): pid=25 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:00,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=25 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=25 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:00,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/6ef21b673c3545ee96acb63cc5aa2bbc 2023-09-06T20:14:00,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/7388978ff3854364b2f5ef1a8b41b62f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/7388978ff3854364b2f5ef1a8b41b62f 2023-09-06T20:14:00,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/7388978ff3854364b2f5ef1a8b41b62f, entries=250, sequenceid=501, filesize=16.8 K 2023-09-06T20:14:00,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/f990f5bf9f6d419ba3aca9aefb8eabfb as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/f990f5bf9f6d419ba3aca9aefb8eabfb 2023-09-06T20:14:00,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/f990f5bf9f6d419ba3aca9aefb8eabfb, entries=150, sequenceid=501, filesize=12.0 K 2023-09-06T20:14:00,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/6ef21b673c3545ee96acb63cc5aa2bbc as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6ef21b673c3545ee96acb63cc5aa2bbc 2023-09-06T20:14:00,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6ef21b673c3545ee96acb63cc5aa2bbc, entries=150, sequenceid=501, filesize=12.0 K 2023-09-06T20:14:00,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=26.84 KB/27480 for a659f678dfa70a1da8b63a91043e10d5 in 920ms, sequenceid=501, compaction requested=true 2023-09-06T20:14:00,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:00,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:00,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:00,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:00,584 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:00,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:00,584 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:00,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:00,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:00,585 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40378 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:00,586 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/A is initiating minor compaction (all files) 2023-09-06T20:14:00,586 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/A in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:00,586 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/fa89ed69f19c4d50bf0b3b81d246b993, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/5b5edbf252d745ab9dd571c676aee69a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/7388978ff3854364b2f5ef1a8b41b62f] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=39.4 K 2023-09-06T20:14:00,586 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47714 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:00,586 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/B is initiating minor compaction (all files) 2023-09-06T20:14:00,586 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/B in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:00,586 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/83cc07450b994b7a87642390901b9897, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/4045171879314bdb870b75bc9205333b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d0fce4b1be0f4239a3d652b4128541a6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/f990f5bf9f6d419ba3aca9aefb8eabfb] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=46.6 K 2023-09-06T20:14:00,587 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting fa89ed69f19c4d50bf0b3b81d246b993, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1694031238188 2023-09-06T20:14:00,587 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5b5edbf252d745ab9dd571c676aee69a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1694031238197 2023-09-06T20:14:00,588 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7388978ff3854364b2f5ef1a8b41b62f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1694031238539 2023-09-06T20:14:00,589 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 83cc07450b994b7a87642390901b9897, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1694031238044 2023-09-06T20:14:00,592 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 4045171879314bdb870b75bc9205333b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1694031238188 2023-09-06T20:14:00,593 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting d0fce4b1be0f4239a3d652b4128541a6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1694031238197 2023-09-06T20:14:00,593 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting f990f5bf9f6d419ba3aca9aefb8eabfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1694031238539 2023-09-06T20:14:00,607 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#A#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:00,608 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/045c0309144c429188d1d013abb848e0 is 50, key is test_row_0/A:col10/1694031238542/Put/seqid=0 2023-09-06T20:14:00,612 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#B#compaction#104 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:00,612 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/3174088b15504357a5c06d698343980b is 50, key is test_row_0/B:col10/1694031238542/Put/seqid=0 2023-09-06T20:14:00,654 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/045c0309144c429188d1d013abb848e0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/045c0309144c429188d1d013abb848e0 2023-09-06T20:14:00,656 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/3174088b15504357a5c06d698343980b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/3174088b15504357a5c06d698343980b 2023-09-06T20:14:00,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2023-09-06T20:14:00,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:00,657 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2023-09-06T20:14:00,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:14:00,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:00,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:14:00,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:00,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:14:00,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:00,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/4a679efcd18944038d2799e00f27856d is 50, key is test_row_1/A:col10/1694031239669/Put/seqid=0 2023-09-06T20:14:00,678 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/4a679efcd18944038d2799e00f27856d 2023-09-06T20:14:00,680 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/B of a659f678dfa70a1da8b63a91043e10d5 into 3174088b15504357a5c06d698343980b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:00,680 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:00,680 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/B, priority=12, startTime=1694031240584; duration=0sec 2023-09-06T20:14:00,680 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:00,680 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:14:00,680 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:00,681 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/A of a659f678dfa70a1da8b63a91043e10d5 into 045c0309144c429188d1d013abb848e0(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:00,682 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:00,682 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/A, priority=13, startTime=1694031240584; duration=0sec 2023-09-06T20:14:00,682 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:00,682 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:14:00,683 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47714 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:00,683 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/C is initiating minor compaction (all files) 2023-09-06T20:14:00,683 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/C in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:00,683 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/310a78ec80fd4f139b75f98a9ef6cfdb, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b6a7c58e47c44224b222c5547ac431df, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/28760f2b3e5b46f1bbfa2b3272e7683f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6ef21b673c3545ee96acb63cc5aa2bbc] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=46.6 K 2023-09-06T20:14:00,684 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 310a78ec80fd4f139b75f98a9ef6cfdb, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1694031238044 2023-09-06T20:14:00,684 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b6a7c58e47c44224b222c5547ac431df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1694031238188 2023-09-06T20:14:00,685 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 28760f2b3e5b46f1bbfa2b3272e7683f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1694031238197 2023-09-06T20:14:00,685 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 6ef21b673c3545ee96acb63cc5aa2bbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1694031238539 2023-09-06T20:14:00,714 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#C#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:00,715 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/01e2200249744ca4a047f26824ae87c6 is 50, key is test_row_0/C:col10/1694031238542/Put/seqid=0 2023-09-06T20:14:00,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/f84d75c9f9b94f6292c775256dbe3848 is 50, key is test_row_1/B:col10/1694031239669/Put/seqid=0 2023-09-06T20:14:00,730 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/f84d75c9f9b94f6292c775256dbe3848 2023-09-06T20:14:00,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/0f2df07bcd5c498a83ae9b8d8ea08e3a is 50, key is test_row_1/C:col10/1694031239669/Put/seqid=0 2023-09-06T20:14:00,751 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/01e2200249744ca4a047f26824ae87c6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/01e2200249744ca4a047f26824ae87c6 2023-09-06T20:14:00,759 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/C of a659f678dfa70a1da8b63a91043e10d5 into 01e2200249744ca4a047f26824ae87c6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:00,759 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:00,759 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/C, priority=12, startTime=1694031240584; duration=0sec 2023-09-06T20:14:00,760 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:00,760 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:14:00,769 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/0f2df07bcd5c498a83ae9b8d8ea08e3a 2023-09-06T20:14:00,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/4a679efcd18944038d2799e00f27856d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/4a679efcd18944038d2799e00f27856d 2023-09-06T20:14:00,782 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/4a679efcd18944038d2799e00f27856d, entries=50, sequenceid=508, filesize=7.2 K 2023-09-06T20:14:00,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/f84d75c9f9b94f6292c775256dbe3848 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/f84d75c9f9b94f6292c775256dbe3848 2023-09-06T20:14:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:14:00,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:14:00,791 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/f84d75c9f9b94f6292c775256dbe3848, entries=50, sequenceid=508, filesize=7.2 K 2023-09-06T20:14:00,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/0f2df07bcd5c498a83ae9b8d8ea08e3a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0f2df07bcd5c498a83ae9b8d8ea08e3a 2023-09-06T20:14:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-09-06T20:14:00,798 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0f2df07bcd5c498a83ae9b8d8ea08e3a, entries=50, sequenceid=508, filesize=7.2 K 2023-09-06T20:14:00,833 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(3022): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=40.25 KB/41220 for a659f678dfa70a1da8b63a91043e10d5 in 175ms, sequenceid=508, compaction requested=false 2023-09-06T20:14:00,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:00,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:00,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=25 2023-09-06T20:14:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:14:00,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:14:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=25 2023-09-06T20:14:00,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:14:00,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:00,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:14:00,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:00,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:14:00,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:00,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/ab64ad37cc8b4c66b83fbdad655e901b is 50, key is test_row_0/A:col10/1694031240788/Put/seqid=0 2023-09-06T20:14:00,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=25, resume processing ppid=24 2023-09-06T20:14:00,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, ppid=24, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 647 msec 2023-09-06T20:14:00,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees in 658 msec 2023-09-06T20:14:00,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031300864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031300864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031300865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031300865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031300866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/ab64ad37cc8b4c66b83fbdad655e901b 2023-09-06T20:14:00,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/fb418dd4f023490193c26d9207401d37 is 50, key is test_row_0/B:col10/1694031240788/Put/seqid=0 2023-09-06T20:14:00,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/fb418dd4f023490193c26d9207401d37 2023-09-06T20:14:00,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/bb64bae323ed41f89fc25faa8cc2952d is 50, key is test_row_0/C:col10/1694031240788/Put/seqid=0 2023-09-06T20:14:00,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031300968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031300968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031300969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031300969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:00,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031300972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/bb64bae323ed41f89fc25faa8cc2952d 2023-09-06T20:14:01,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/ab64ad37cc8b4c66b83fbdad655e901b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ab64ad37cc8b4c66b83fbdad655e901b 2023-09-06T20:14:01,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ab64ad37cc8b4c66b83fbdad655e901b, entries=250, sequenceid=525, filesize=16.8 K 2023-09-06T20:14:01,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/fb418dd4f023490193c26d9207401d37 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/fb418dd4f023490193c26d9207401d37 2023-09-06T20:14:01,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/fb418dd4f023490193c26d9207401d37, entries=150, sequenceid=525, filesize=12.0 K 2023-09-06T20:14:01,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/bb64bae323ed41f89fc25faa8cc2952d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/bb64bae323ed41f89fc25faa8cc2952d 2023-09-06T20:14:01,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/bb64bae323ed41f89fc25faa8cc2952d, entries=150, sequenceid=525, filesize=12.0 K 2023-09-06T20:14:01,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a659f678dfa70a1da8b63a91043e10d5 in 201ms, sequenceid=525, compaction requested=true 2023-09-06T20:14:01,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:01,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:01,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:01,035 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:01,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:01,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:01,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:01,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-06T20:14:01,035 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:01,036 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38038 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:01,037 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/A is initiating minor compaction (all files) 2023-09-06T20:14:01,037 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/A in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:01,037 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/045c0309144c429188d1d013abb848e0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/4a679efcd18944038d2799e00f27856d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ab64ad37cc8b4c66b83fbdad655e901b] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=37.1 K 2023-09-06T20:14:01,037 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33124 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:01,037 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/B is initiating minor compaction (all files) 2023-09-06T20:14:01,037 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/B in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:01,037 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/3174088b15504357a5c06d698343980b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/f84d75c9f9b94f6292c775256dbe3848, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/fb418dd4f023490193c26d9207401d37] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=32.3 K 2023-09-06T20:14:01,038 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 045c0309144c429188d1d013abb848e0, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1694031238539 2023-09-06T20:14:01,038 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4a679efcd18944038d2799e00f27856d, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1694031239669 2023-09-06T20:14:01,039 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 3174088b15504357a5c06d698343980b, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1694031238539 2023-09-06T20:14:01,039 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting ab64ad37cc8b4c66b83fbdad655e901b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1694031240788 2023-09-06T20:14:01,040 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting f84d75c9f9b94f6292c775256dbe3848, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1694031239669 2023-09-06T20:14:01,041 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting fb418dd4f023490193c26d9207401d37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1694031240788 2023-09-06T20:14:01,059 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#A#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:01,060 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/8cf3d39e21c64961a2c4b3032b6447f9 is 50, key is test_row_0/A:col10/1694031240788/Put/seqid=0 2023-09-06T20:14:01,070 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#B#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:01,070 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/88160ceb30d541b2becb2b2e6a2af52d is 50, key is test_row_0/B:col10/1694031240788/Put/seqid=0 2023-09-06T20:14:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:14:01,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-06T20:14:01,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:14:01,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:01,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:14:01,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:01,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:14:01,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:01,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/403d99ede96c457ea96d4d9a252b1eed is 50, key is test_row_0/A:col10/1694031240863/Put/seqid=0 2023-09-06T20:14:01,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/403d99ede96c457ea96d4d9a252b1eed 2023-09-06T20:14:01,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031301187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031301187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031301188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031301189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031301189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/0a24ffebf24f4f7198b8d0a7b80db0d6 is 50, key is test_row_0/B:col10/1694031240863/Put/seqid=0 2023-09-06T20:14:01,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/0a24ffebf24f4f7198b8d0a7b80db0d6 2023-09-06T20:14:01,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/5c1d3908e3f349ceabc3f57badd077d7 is 50, key is test_row_0/C:col10/1694031240863/Put/seqid=0 2023-09-06T20:14:01,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031301293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031301293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-09-06T20:14:01,296 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 24 completed 2023-09-06T20:14:01,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031301296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031301297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees 2023-09-06T20:14:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:14:01,301 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=26, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:01,301 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=26, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:01,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:01,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031301297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:14:01,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2023-09-06T20:14:01,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:01,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:14:01,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:01,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(53): pid=27 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:01,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:01,493 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/8cf3d39e21c64961a2c4b3032b6447f9 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/8cf3d39e21c64961a2c4b3032b6447f9 2023-09-06T20:14:01,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031301497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031301498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031301498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031301499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,500 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/A of a659f678dfa70a1da8b63a91043e10d5 into 8cf3d39e21c64961a2c4b3032b6447f9(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:01,500 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:01,500 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/A, priority=13, startTime=1694031241034; duration=0sec 2023-09-06T20:14:01,500 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:01,500 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:14:01,500 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:01,502 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33124 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:01,502 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/C is initiating minor compaction (all files) 2023-09-06T20:14:01,502 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/C in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:01,502 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/01e2200249744ca4a047f26824ae87c6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0f2df07bcd5c498a83ae9b8d8ea08e3a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/bb64bae323ed41f89fc25faa8cc2952d] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=32.3 K 2023-09-06T20:14:01,503 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 01e2200249744ca4a047f26824ae87c6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1694031238539 2023-09-06T20:14:01,506 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 0f2df07bcd5c498a83ae9b8d8ea08e3a, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1694031239669 2023-09-06T20:14:01,512 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting bb64bae323ed41f89fc25faa8cc2952d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1694031240788 2023-09-06T20:14:01,512 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/88160ceb30d541b2becb2b2e6a2af52d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/88160ceb30d541b2becb2b2e6a2af52d 2023-09-06T20:14:01,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031301510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,520 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/B of a659f678dfa70a1da8b63a91043e10d5 into 88160ceb30d541b2becb2b2e6a2af52d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:01,520 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:01,520 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/B, priority=13, startTime=1694031241035; duration=0sec 2023-09-06T20:14:01,521 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:01,521 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:14:01,527 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#C#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:01,528 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/8b28ad9a428647df8dc61b0d86aefc78 is 50, key is test_row_0/C:col10/1694031240788/Put/seqid=0 2023-09-06T20:14:01,538 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/8b28ad9a428647df8dc61b0d86aefc78 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/8b28ad9a428647df8dc61b0d86aefc78 2023-09-06T20:14:01,543 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/C of a659f678dfa70a1da8b63a91043e10d5 into 8b28ad9a428647df8dc61b0d86aefc78(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:01,544 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:01,544 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/C, priority=13, startTime=1694031241035; duration=0sec 2023-09-06T20:14:01,544 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:01,544 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:14:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:14:01,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2023-09-06T20:14:01,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:01,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:14:01,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:01,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(53): pid=27 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:01,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:01,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/5c1d3908e3f349ceabc3f57badd077d7 2023-09-06T20:14:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/403d99ede96c457ea96d4d9a252b1eed as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/403d99ede96c457ea96d4d9a252b1eed 2023-09-06T20:14:01,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/403d99ede96c457ea96d4d9a252b1eed, entries=200, sequenceid=548, filesize=14.4 K 2023-09-06T20:14:01,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/0a24ffebf24f4f7198b8d0a7b80db0d6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0a24ffebf24f4f7198b8d0a7b80db0d6 2023-09-06T20:14:01,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0a24ffebf24f4f7198b8d0a7b80db0d6, entries=150, sequenceid=548, filesize=12.0 K 2023-09-06T20:14:01,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/5c1d3908e3f349ceabc3f57badd077d7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/5c1d3908e3f349ceabc3f57badd077d7 2023-09-06T20:14:01,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/5c1d3908e3f349ceabc3f57badd077d7, entries=150, sequenceid=548, filesize=12.0 K 2023-09-06T20:14:01,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for a659f678dfa70a1da8b63a91043e10d5 in 490ms, sequenceid=548, compaction requested=false 2023-09-06T20:14:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:01,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2023-09-06T20:14:01,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:01,764 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-06T20:14:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:14:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:14:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:14:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/ae6f7d190eb74b3894cc20caaf922791 is 50, key is test_row_0/A:col10/1694031241178/Put/seqid=0 2023-09-06T20:14:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:14:01,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:14:01,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031301820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031301820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031301820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031301821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031301821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:14:01,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031301925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031301926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031301926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031301928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:01,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031301931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031302128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031302129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031302129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031302133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031302134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,180 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/ae6f7d190eb74b3894cc20caaf922791 2023-09-06T20:14:02,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/49c372ac5e294dbcbc6143d7b23447f2 is 50, key is test_row_0/B:col10/1694031241178/Put/seqid=0 2023-09-06T20:14:02,196 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/49c372ac5e294dbcbc6143d7b23447f2 2023-09-06T20:14:02,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/9096946f28484d14905221253e98ae37 is 50, key is test_row_0/C:col10/1694031241178/Put/seqid=0 2023-09-06T20:14:02,209 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/9096946f28484d14905221253e98ae37 2023-09-06T20:14:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/ae6f7d190eb74b3894cc20caaf922791 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ae6f7d190eb74b3894cc20caaf922791 2023-09-06T20:14:02,219 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ae6f7d190eb74b3894cc20caaf922791, entries=150, sequenceid=564, filesize=12.0 K 2023-09-06T20:14:02,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/49c372ac5e294dbcbc6143d7b23447f2 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/49c372ac5e294dbcbc6143d7b23447f2 2023-09-06T20:14:02,225 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/49c372ac5e294dbcbc6143d7b23447f2, entries=150, sequenceid=564, filesize=12.0 K 2023-09-06T20:14:02,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/9096946f28484d14905221253e98ae37 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9096946f28484d14905221253e98ae37 2023-09-06T20:14:02,236 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9096946f28484d14905221253e98ae37, entries=150, sequenceid=564, filesize=12.0 K 2023-09-06T20:14:02,237 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a659f678dfa70a1da8b63a91043e10d5 in 473ms, sequenceid=564, compaction requested=true 2023-09-06T20:14:02,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:02,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:02,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=27 2023-09-06T20:14:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=27 2023-09-06T20:14:02,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-09-06T20:14:02,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 941 msec 2023-09-06T20:14:02,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees in 946 msec 2023-09-06T20:14:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:14:02,405 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 26 completed 2023-09-06T20:14:02,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=28, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees 2023-09-06T20:14:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=28 2023-09-06T20:14:02,410 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=28, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:02,411 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=28, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:02,411 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:14:02,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-06T20:14:02,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:14:02,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:02,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:14:02,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:02,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:14:02,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:02,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/c2c681a919524683b305050de6c4b356 is 50, key is test_row_0/A:col10/1694031241819/Put/seqid=0 2023-09-06T20:14:02,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031302443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031302444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031302445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031302445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031302445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/c2c681a919524683b305050de6c4b356 2023-09-06T20:14:02,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/51ea84c3062d473db034648d5cc9f1c2 is 50, key is test_row_0/B:col10/1694031241819/Put/seqid=0 2023-09-06T20:14:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=28 2023-09-06T20:14:02,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031302548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031302550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031302550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031302550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031302551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2023-09-06T20:14:02,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:02,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:14:02,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:02,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(53): pid=29 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:02,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:02,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=28 2023-09-06T20:14:02,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2023-09-06T20:14:02,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:02,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:14:02,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:02,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(53): pid=29 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:02,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:02,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031302751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031302753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031302754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031302754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031302754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:02,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2023-09-06T20:14:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:14:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:02,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(53): pid=29 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:02,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/51ea84c3062d473db034648d5cc9f1c2 2023-09-06T20:14:02,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/91ded994d92b4af9915efb216bb6010e is 50, key is test_row_0/C:col10/1694031241819/Put/seqid=0 2023-09-06T20:14:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=28 2023-09-06T20:14:03,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2023-09-06T20:14:03,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:03,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:14:03,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:03,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(53): pid=29 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:03,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:03,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36022 deadline: 1694031303054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:03,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35968 deadline: 1694031303055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:03,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:35984 deadline: 1694031303057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:03,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36012 deadline: 1694031303058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:03,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:36016 deadline: 1694031303059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:03,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2023-09-06T20:14:03,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:03,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:14:03,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:03,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(53): pid=29 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:03,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:03,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/91ded994d92b4af9915efb216bb6010e 2023-09-06T20:14:03,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/c2c681a919524683b305050de6c4b356 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c2c681a919524683b305050de6c4b356 2023-09-06T20:14:03,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c2c681a919524683b305050de6c4b356, entries=150, sequenceid=590, filesize=12.0 K 2023-09-06T20:14:03,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/51ea84c3062d473db034648d5cc9f1c2 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/51ea84c3062d473db034648d5cc9f1c2 2023-09-06T20:14:03,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/51ea84c3062d473db034648d5cc9f1c2, entries=150, sequenceid=590, filesize=12.0 K 2023-09-06T20:14:03,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/91ded994d92b4af9915efb216bb6010e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/91ded994d92b4af9915efb216bb6010e 2023-09-06T20:14:03,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2023-09-06T20:14:03,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:14:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:03,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(53): pid=29 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:03,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/91ded994d92b4af9915efb216bb6010e, entries=150, sequenceid=590, filesize=12.0 K 2023-09-06T20:14:03,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for a659f678dfa70a1da8b63a91043e10d5 in 900ms, sequenceid=590, compaction requested=true 2023-09-06T20:14:03,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:03,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:03,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:03,335 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:03,335 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:03,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:03,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:03,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a659f678dfa70a1da8b63a91043e10d5:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:03,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:03,339 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52870 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:03,339 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/A is initiating minor compaction (all files) 2023-09-06T20:14:03,339 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:03,339 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/B is initiating minor compaction (all files) 2023-09-06T20:14:03,339 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/A in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:03,339 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/B in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:03,339 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/8cf3d39e21c64961a2c4b3032b6447f9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/403d99ede96c457ea96d4d9a252b1eed, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ae6f7d190eb74b3894cc20caaf922791, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c2c681a919524683b305050de6c4b356] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=51.6 K 2023-09-06T20:14:03,339 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/88160ceb30d541b2becb2b2e6a2af52d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0a24ffebf24f4f7198b8d0a7b80db0d6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/49c372ac5e294dbcbc6143d7b23447f2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/51ea84c3062d473db034648d5cc9f1c2] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=49.2 K 2023-09-06T20:14:03,340 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8cf3d39e21c64961a2c4b3032b6447f9, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1694031240788 2023-09-06T20:14:03,340 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 88160ceb30d541b2becb2b2e6a2af52d, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1694031240788 2023-09-06T20:14:03,340 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 403d99ede96c457ea96d4d9a252b1eed, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1694031240861 2023-09-06T20:14:03,341 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 0a24ffebf24f4f7198b8d0a7b80db0d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1694031240863 2023-09-06T20:14:03,341 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting ae6f7d190eb74b3894cc20caaf922791, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1694031241178 2023-09-06T20:14:03,341 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 49c372ac5e294dbcbc6143d7b23447f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1694031241178 2023-09-06T20:14:03,342 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting c2c681a919524683b305050de6c4b356, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1694031241819 2023-09-06T20:14:03,342 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 51ea84c3062d473db034648d5cc9f1c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1694031241819 2023-09-06T20:14:03,360 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#A#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:03,360 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#B#compaction#125 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:03,360 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/f908f8c1cf3748808589a92595d560eb is 50, key is test_row_0/A:col10/1694031241819/Put/seqid=0 2023-09-06T20:14:03,361 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/0789f02b14bb4a59a90a94fb9faff6c5 is 50, key is test_row_0/B:col10/1694031241819/Put/seqid=0 2023-09-06T20:14:03,375 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/f908f8c1cf3748808589a92595d560eb as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/f908f8c1cf3748808589a92595d560eb 2023-09-06T20:14:03,382 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/A of a659f678dfa70a1da8b63a91043e10d5 into f908f8c1cf3748808589a92595d560eb(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:03,382 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:03,382 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/A, priority=12, startTime=1694031243335; duration=0sec 2023-09-06T20:14:03,382 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:03,382 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:A 2023-09-06T20:14:03,382 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:03,384 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:03,384 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): a659f678dfa70a1da8b63a91043e10d5/C is initiating minor compaction (all files) 2023-09-06T20:14:03,384 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of a659f678dfa70a1da8b63a91043e10d5/C in TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:03,384 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/8b28ad9a428647df8dc61b0d86aefc78, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/5c1d3908e3f349ceabc3f57badd077d7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9096946f28484d14905221253e98ae37, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/91ded994d92b4af9915efb216bb6010e] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp, totalSize=49.2 K 2023-09-06T20:14:03,385 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8b28ad9a428647df8dc61b0d86aefc78, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1694031240788 2023-09-06T20:14:03,385 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5c1d3908e3f349ceabc3f57badd077d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1694031240863 2023-09-06T20:14:03,386 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9096946f28484d14905221253e98ae37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1694031241178 2023-09-06T20:14:03,387 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 91ded994d92b4af9915efb216bb6010e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1694031241819 2023-09-06T20:14:03,397 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a659f678dfa70a1da8b63a91043e10d5#C#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:03,398 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/85c3e97d55104857abcc793af44eb6d6 is 50, key is test_row_0/C:col10/1694031241819/Put/seqid=0 2023-09-06T20:14:03,407 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/85c3e97d55104857abcc793af44eb6d6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/85c3e97d55104857abcc793af44eb6d6 2023-09-06T20:14:03,414 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/C of a659f678dfa70a1da8b63a91043e10d5 into 85c3e97d55104857abcc793af44eb6d6(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:03,414 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:03,414 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/C, priority=12, startTime=1694031243336; duration=0sec 2023-09-06T20:14:03,415 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:03,415 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:C 2023-09-06T20:14:03,439 DEBUG [Thread-155 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x190efedc to 127.0.0.1:58481 2023-09-06T20:14:03,439 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:03,440 DEBUG [Thread-153 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x11920f5c to 127.0.0.1:58481 2023-09-06T20:14:03,440 DEBUG [Thread-153 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:03,443 DEBUG [Thread-159 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2d7329e4 to 127.0.0.1:58481 2023-09-06T20:14:03,443 DEBUG [Thread-159 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:03,443 DEBUG [Thread-157 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x57847d41 to 127.0.0.1:58481 2023-09-06T20:14:03,443 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:03,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2023-09-06T20:14:03,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:03,486 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:14:03,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:14:03,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:03,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:14:03,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:03,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:14:03,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:03,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/52a7abef277a42d992b9bcb658e33b67 is 50, key is test_row_0/A:col10/1694031242443/Put/seqid=0 2023-09-06T20:14:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=28 2023-09-06T20:14:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:14:03,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. as already flushing 2023-09-06T20:14:03,560 DEBUG [Thread-148 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2be9788f to 127.0.0.1:58481 2023-09-06T20:14:03,560 DEBUG [Thread-148 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:03,562 DEBUG [Thread-146 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1fb807fe to 127.0.0.1:58481 2023-09-06T20:14:03,562 DEBUG [Thread-146 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:03,563 DEBUG [Thread-150 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x77c03710 to 127.0.0.1:58481 2023-09-06T20:14:03,563 DEBUG [Thread-150 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:03,565 DEBUG [Thread-142 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3bd38762 to 127.0.0.1:58481 2023-09-06T20:14:03,565 DEBUG [Thread-142 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:03,565 DEBUG [Thread-144 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4c616450 to 127.0.0.1:58481 2023-09-06T20:14:03,565 DEBUG [Thread-144 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:03,771 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/0789f02b14bb4a59a90a94fb9faff6c5 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0789f02b14bb4a59a90a94fb9faff6c5 2023-09-06T20:14:03,777 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in a659f678dfa70a1da8b63a91043e10d5/B of a659f678dfa70a1da8b63a91043e10d5 into 0789f02b14bb4a59a90a94fb9faff6c5(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:03,777 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:03,777 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5., storeName=a659f678dfa70a1da8b63a91043e10d5/B, priority=12, startTime=1694031243335; duration=0sec 2023-09-06T20:14:03,777 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:03,777 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a659f678dfa70a1da8b63a91043e10d5:B 2023-09-06T20:14:03,900 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/52a7abef277a42d992b9bcb658e33b67 2023-09-06T20:14:03,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/6fba37b1a20440498c1126da3250e350 is 50, key is test_row_0/B:col10/1694031242443/Put/seqid=0 2023-09-06T20:14:03,924 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/6fba37b1a20440498c1126da3250e350 2023-09-06T20:14:03,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/6dbb0866af554992ba5b1608ccfe25eb is 50, key is test_row_0/C:col10/1694031242443/Put/seqid=0 2023-09-06T20:14:04,342 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/6dbb0866af554992ba5b1608ccfe25eb 2023-09-06T20:14:04,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/52a7abef277a42d992b9bcb658e33b67 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/52a7abef277a42d992b9bcb658e33b67 2023-09-06T20:14:04,352 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/52a7abef277a42d992b9bcb658e33b67, entries=150, sequenceid=604, filesize=12.0 K 2023-09-06T20:14:04,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/6fba37b1a20440498c1126da3250e350 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/6fba37b1a20440498c1126da3250e350 2023-09-06T20:14:04,358 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/6fba37b1a20440498c1126da3250e350, entries=150, sequenceid=604, filesize=12.0 K 2023-09-06T20:14:04,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/6dbb0866af554992ba5b1608ccfe25eb as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6dbb0866af554992ba5b1608ccfe25eb 2023-09-06T20:14:04,364 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6dbb0866af554992ba5b1608ccfe25eb, entries=150, sequenceid=604, filesize=12.0 K 2023-09-06T20:14:04,365 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=33.54 KB/34350 for a659f678dfa70a1da8b63a91043e10d5 in 878ms, sequenceid=604, compaction requested=false 2023-09-06T20:14:04,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2520): Flush status journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:04,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:04,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=29 2023-09-06T20:14:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=29 2023-09-06T20:14:04,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=29, resume processing ppid=28 2023-09-06T20:14:04,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, ppid=28, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9550 sec 2023-09-06T20:14:04,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees in 1.9620 sec 2023-09-06T20:14:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=28 2023-09-06T20:14:04,513 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 28 completed 2023-09-06T20:14:04,514 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2023-09-06T20:14:04,514 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2023-09-06T20:14:04,514 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2023-09-06T20:14:04,514 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2023-09-06T20:14:04,514 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2023-09-06T20:14:04,514 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2023-09-06T20:14:04,514 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2023-09-06T20:14:04,514 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(398): read 5694 2023-09-06T20:14:04,514 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(398): read 5362 2023-09-06T20:14:04,514 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2023-09-06T20:14:04,514 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(402): scanned 2607 2023-09-06T20:14:04,514 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(403): verified 7820 rows 2023-09-06T20:14:04,514 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(402): scanned 2619 2023-09-06T20:14:04,514 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(403): verified 7847 rows 2023-09-06T20:14:04,515 INFO [Listener at localhost/35961 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:14:04,515 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6d771096 to 127.0.0.1:58481 2023-09-06T20:14:04,515 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:04,523 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2023-09-06T20:14:04,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable TestAcidGuarantees 2023-09-06T20:14:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=30, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:04,538 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031244538"}]},"ts":"1694031244538"} 2023-09-06T20:14:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-06T20:14:04,540 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2023-09-06T20:14:04,622 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2023-09-06T20:14:04,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=a659f678dfa70a1da8b63a91043e10d5, UNASSIGN}] 2023-09-06T20:14:04,625 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=a659f678dfa70a1da8b63a91043e10d5, UNASSIGN 2023-09-06T20:14:04,626 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=a659f678dfa70a1da8b63a91043e10d5, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:04,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; CloseRegionProcedure a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:14:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-06T20:14:04,786 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(119): Close a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:14:04,786 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1668): Closing a659f678dfa70a1da8b63a91043e10d5, disabling compactions & flushes 2023-09-06T20:14:04,786 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:04,786 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:04,786 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. after waiting 0 ms 2023-09-06T20:14:04,786 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:04,786 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2819): Flushing a659f678dfa70a1da8b63a91043e10d5 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2023-09-06T20:14:04,786 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=A 2023-09-06T20:14:04,787 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:04,787 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=B 2023-09-06T20:14:04,787 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:04,787 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(204): FLUSHING TO DISK a659f678dfa70a1da8b63a91043e10d5, store=C 2023-09-06T20:14:04,787 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:04,790 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/863794d8e3474d688a6d7295b8f97b38 is 50, key is test_row_0/A:col10/1694031243564/Put/seqid=0 2023-09-06T20:14:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-06T20:14:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-06T20:14:05,195 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/863794d8e3474d688a6d7295b8f97b38 2023-09-06T20:14:05,207 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/d3e79742d10346eb8d618fa25439104a is 50, key is test_row_0/B:col10/1694031243564/Put/seqid=0 2023-09-06T20:14:05,211 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/d3e79742d10346eb8d618fa25439104a 2023-09-06T20:14:05,218 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/51d1a687fbd94e069a630f3dc172e89e is 50, key is test_row_0/C:col10/1694031243564/Put/seqid=0 2023-09-06T20:14:05,622 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/51d1a687fbd94e069a630f3dc172e89e 2023-09-06T20:14:05,628 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/A/863794d8e3474d688a6d7295b8f97b38 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/863794d8e3474d688a6d7295b8f97b38 2023-09-06T20:14:05,633 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/863794d8e3474d688a6d7295b8f97b38, entries=100, sequenceid=613, filesize=9.6 K 2023-09-06T20:14:05,634 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/B/d3e79742d10346eb8d618fa25439104a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d3e79742d10346eb8d618fa25439104a 2023-09-06T20:14:05,638 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d3e79742d10346eb8d618fa25439104a, entries=100, sequenceid=613, filesize=9.6 K 2023-09-06T20:14:05,639 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/.tmp/C/51d1a687fbd94e069a630f3dc172e89e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/51d1a687fbd94e069a630f3dc172e89e 2023-09-06T20:14:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-06T20:14:05,646 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/51d1a687fbd94e069a630f3dc172e89e, entries=100, sequenceid=613, filesize=9.6 K 2023-09-06T20:14:05,647 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3022): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a659f678dfa70a1da8b63a91043e10d5 in 861ms, sequenceid=613, compaction requested=true 2023-09-06T20:14:05,671 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/81f21d3db4e64aefa74de166313ffa6b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/6ee081991ab94dc5b4d4d4feed7f0a1f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/642fe0fe86a040fba5ca9046fd2d923f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/63fcd3ca58bc4d109807677a646de71e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/603c67e5f3f64a4087dbea4de88a79a6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/e417bfcb98734ce08aca3ea43b2f8218, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/f9914ec9c7be456fb1a22e10b0ed09a7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2938d16a54e644019d0d8407b7d8da0b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/83b8ad16f2d946319994542da94c178d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/06bafdf833464e04bd8a1aebbdceae4b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c8af0a32c2d3462dabda3417035c9dfe, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/d8ababdfea1e4340b03070dbf4e3aaab, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2d2aedd8be4a441983339846f91169c0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/73a4f7ebe0a342afb640a5b0bd4e2f95, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/dba4da29b33642f2857f86492930ce22, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cfc4e7f1e32a453299de83b81e753273, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c1370773997b48ee809db433daa58c25, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/d90aab3f12154179bc7890bfd5f5c395, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cac999c88e5c481e83e82cc7f18724b2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/1792e24106d04db39b74d471a1d4800a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cc115800d3814b4d86e6d75dd4ef09ab, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/a8f5409b1d9c47948718cdb7f165e4f3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ee80b7a5350c4ab6a07916ad7e14c7e5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/1b04f285f2734436835df3d5ecea5d7d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/695f38e7890840809455ac65df5f56fa, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/467d6eb805f04b829653e5a55b81dda3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/b2eb83c4e9724754af6537f7af0f0fe6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/aab3ca2517cf4d1ba1bcbdc823295073, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/84b71622ebaf4c9bac9a1a749ea1376a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/b08f4500565342f18fdd9c2c483631ca, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/245a742276e14a158efcd33dd9284ac6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2471e1d6348b460cb9f7ae300e144ff7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/fa89ed69f19c4d50bf0b3b81d246b993, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/5b5edbf252d745ab9dd571c676aee69a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/7388978ff3854364b2f5ef1a8b41b62f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/045c0309144c429188d1d013abb848e0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/4a679efcd18944038d2799e00f27856d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ab64ad37cc8b4c66b83fbdad655e901b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/8cf3d39e21c64961a2c4b3032b6447f9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/403d99ede96c457ea96d4d9a252b1eed, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ae6f7d190eb74b3894cc20caaf922791, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c2c681a919524683b305050de6c4b356] to archive 2023-09-06T20:14:05,675 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:14:05,682 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/81f21d3db4e64aefa74de166313ffa6b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/81f21d3db4e64aefa74de166313ffa6b 2023-09-06T20:14:05,684 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/6ee081991ab94dc5b4d4d4feed7f0a1f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/6ee081991ab94dc5b4d4d4feed7f0a1f 2023-09-06T20:14:05,686 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/642fe0fe86a040fba5ca9046fd2d923f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/642fe0fe86a040fba5ca9046fd2d923f 2023-09-06T20:14:05,687 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/63fcd3ca58bc4d109807677a646de71e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/63fcd3ca58bc4d109807677a646de71e 2023-09-06T20:14:05,689 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/603c67e5f3f64a4087dbea4de88a79a6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/603c67e5f3f64a4087dbea4de88a79a6 2023-09-06T20:14:05,690 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/e417bfcb98734ce08aca3ea43b2f8218 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/e417bfcb98734ce08aca3ea43b2f8218 2023-09-06T20:14:05,692 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/f9914ec9c7be456fb1a22e10b0ed09a7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/f9914ec9c7be456fb1a22e10b0ed09a7 2023-09-06T20:14:05,693 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2938d16a54e644019d0d8407b7d8da0b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2938d16a54e644019d0d8407b7d8da0b 2023-09-06T20:14:05,695 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/83b8ad16f2d946319994542da94c178d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/83b8ad16f2d946319994542da94c178d 2023-09-06T20:14:05,697 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/06bafdf833464e04bd8a1aebbdceae4b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/06bafdf833464e04bd8a1aebbdceae4b 2023-09-06T20:14:05,698 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c8af0a32c2d3462dabda3417035c9dfe to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c8af0a32c2d3462dabda3417035c9dfe 2023-09-06T20:14:05,700 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/d8ababdfea1e4340b03070dbf4e3aaab to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/d8ababdfea1e4340b03070dbf4e3aaab 2023-09-06T20:14:05,701 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2d2aedd8be4a441983339846f91169c0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2d2aedd8be4a441983339846f91169c0 2023-09-06T20:14:05,702 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/73a4f7ebe0a342afb640a5b0bd4e2f95 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/73a4f7ebe0a342afb640a5b0bd4e2f95 2023-09-06T20:14:05,704 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/dba4da29b33642f2857f86492930ce22 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/dba4da29b33642f2857f86492930ce22 2023-09-06T20:14:05,705 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cfc4e7f1e32a453299de83b81e753273 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cfc4e7f1e32a453299de83b81e753273 2023-09-06T20:14:05,706 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c1370773997b48ee809db433daa58c25 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c1370773997b48ee809db433daa58c25 2023-09-06T20:14:05,708 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/d90aab3f12154179bc7890bfd5f5c395 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/d90aab3f12154179bc7890bfd5f5c395 2023-09-06T20:14:05,709 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cac999c88e5c481e83e82cc7f18724b2 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cac999c88e5c481e83e82cc7f18724b2 2023-09-06T20:14:05,710 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/1792e24106d04db39b74d471a1d4800a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/1792e24106d04db39b74d471a1d4800a 2023-09-06T20:14:05,712 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cc115800d3814b4d86e6d75dd4ef09ab to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/cc115800d3814b4d86e6d75dd4ef09ab 2023-09-06T20:14:05,713 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/a8f5409b1d9c47948718cdb7f165e4f3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/a8f5409b1d9c47948718cdb7f165e4f3 2023-09-06T20:14:05,714 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ee80b7a5350c4ab6a07916ad7e14c7e5 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ee80b7a5350c4ab6a07916ad7e14c7e5 2023-09-06T20:14:05,715 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/1b04f285f2734436835df3d5ecea5d7d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/1b04f285f2734436835df3d5ecea5d7d 2023-09-06T20:14:05,716 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/695f38e7890840809455ac65df5f56fa to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/695f38e7890840809455ac65df5f56fa 2023-09-06T20:14:05,718 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/467d6eb805f04b829653e5a55b81dda3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/467d6eb805f04b829653e5a55b81dda3 2023-09-06T20:14:05,719 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/b2eb83c4e9724754af6537f7af0f0fe6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/b2eb83c4e9724754af6537f7af0f0fe6 2023-09-06T20:14:05,720 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/aab3ca2517cf4d1ba1bcbdc823295073 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/aab3ca2517cf4d1ba1bcbdc823295073 2023-09-06T20:14:05,721 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/84b71622ebaf4c9bac9a1a749ea1376a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/84b71622ebaf4c9bac9a1a749ea1376a 2023-09-06T20:14:05,723 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/b08f4500565342f18fdd9c2c483631ca to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/b08f4500565342f18fdd9c2c483631ca 2023-09-06T20:14:05,724 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/245a742276e14a158efcd33dd9284ac6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/245a742276e14a158efcd33dd9284ac6 2023-09-06T20:14:05,725 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2471e1d6348b460cb9f7ae300e144ff7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/2471e1d6348b460cb9f7ae300e144ff7 2023-09-06T20:14:05,726 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/fa89ed69f19c4d50bf0b3b81d246b993 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/fa89ed69f19c4d50bf0b3b81d246b993 2023-09-06T20:14:05,727 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/5b5edbf252d745ab9dd571c676aee69a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/5b5edbf252d745ab9dd571c676aee69a 2023-09-06T20:14:05,728 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/7388978ff3854364b2f5ef1a8b41b62f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/7388978ff3854364b2f5ef1a8b41b62f 2023-09-06T20:14:05,729 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/045c0309144c429188d1d013abb848e0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/045c0309144c429188d1d013abb848e0 2023-09-06T20:14:05,731 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/4a679efcd18944038d2799e00f27856d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/4a679efcd18944038d2799e00f27856d 2023-09-06T20:14:05,732 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ab64ad37cc8b4c66b83fbdad655e901b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ab64ad37cc8b4c66b83fbdad655e901b 2023-09-06T20:14:05,733 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/8cf3d39e21c64961a2c4b3032b6447f9 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/8cf3d39e21c64961a2c4b3032b6447f9 2023-09-06T20:14:05,735 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/403d99ede96c457ea96d4d9a252b1eed to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/403d99ede96c457ea96d4d9a252b1eed 2023-09-06T20:14:05,736 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ae6f7d190eb74b3894cc20caaf922791 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/ae6f7d190eb74b3894cc20caaf922791 2023-09-06T20:14:05,738 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c2c681a919524683b305050de6c4b356 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/c2c681a919524683b305050de6c4b356 2023-09-06T20:14:05,770 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/6bb0560e44d04290aeb4945b301f5f1b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/03486942346a46329b8297f9f499f596, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/abeff01a37ae465b8058a323135f094b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b5dda693d5db4638a0f78546cd5520f4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b999c41508784c859fa517dbe5a42f2d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/699ac0859363472cab95ea23b9d07743, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/57467e829d774c89be8ae593ecceafa1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/9b939b787fc54e6086d1ce6ebbc75cb6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/14265a98310f4dc1b384467dd9336c2b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/9977f7217d5a46669317b16af6a035b4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/4e22c69bb7fe41babbc91b5c00660619, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/185bbf9fb325422ab5e56a4c9904dfa7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/61d8abe5ccfd4af394a1ed8fb64f0a1a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0013d9daef4d49a29acf38f78f5dd7a2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/dc22e4c2e4fd4f318b7ad9f2f5080e7f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/803ef76280da4b3fb6fde96a05477566, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d931243386764028ae822b6e272e3f20, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/7ae48a1600464248a09d3beff30f5ef5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/7001ee4fb13047ab8edec7c2d51c6715, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/70e36316f3c84caab5e5128a4192f915, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/258de79a5fc8459daedfc0e3d4db9efa, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/434e2e38a55e4fbe80031b5562f8bc56, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/a5da57ff07e1458cae5f3fe546f3483c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/79a6adcba06441fca93a771003490e8b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/56a40ab8ec4447269501ce3964553bb6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b915b6978c1b40fcbb385ca3baae4cc4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/e545c21e23e8444f805867afb93f1da2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/214e3692091040658d21c57f4ac164f8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/375259bb852344908484384a0a2a4888, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/83cc07450b994b7a87642390901b9897, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/fdd0a209f2984bc3850a7f80ca299e8e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/4045171879314bdb870b75bc9205333b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d0fce4b1be0f4239a3d652b4128541a6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/3174088b15504357a5c06d698343980b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/f990f5bf9f6d419ba3aca9aefb8eabfb, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/f84d75c9f9b94f6292c775256dbe3848, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/88160ceb30d541b2becb2b2e6a2af52d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/fb418dd4f023490193c26d9207401d37, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0a24ffebf24f4f7198b8d0a7b80db0d6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/49c372ac5e294dbcbc6143d7b23447f2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/51ea84c3062d473db034648d5cc9f1c2] to archive 2023-09-06T20:14:05,771 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:14:05,773 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/6bb0560e44d04290aeb4945b301f5f1b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/6bb0560e44d04290aeb4945b301f5f1b 2023-09-06T20:14:05,774 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/03486942346a46329b8297f9f499f596 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/03486942346a46329b8297f9f499f596 2023-09-06T20:14:05,776 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/abeff01a37ae465b8058a323135f094b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/abeff01a37ae465b8058a323135f094b 2023-09-06T20:14:05,777 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b5dda693d5db4638a0f78546cd5520f4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b5dda693d5db4638a0f78546cd5520f4 2023-09-06T20:14:05,778 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b999c41508784c859fa517dbe5a42f2d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b999c41508784c859fa517dbe5a42f2d 2023-09-06T20:14:05,779 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/699ac0859363472cab95ea23b9d07743 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/699ac0859363472cab95ea23b9d07743 2023-09-06T20:14:05,780 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/57467e829d774c89be8ae593ecceafa1 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/57467e829d774c89be8ae593ecceafa1 2023-09-06T20:14:05,781 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/9b939b787fc54e6086d1ce6ebbc75cb6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/9b939b787fc54e6086d1ce6ebbc75cb6 2023-09-06T20:14:05,783 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/14265a98310f4dc1b384467dd9336c2b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/14265a98310f4dc1b384467dd9336c2b 2023-09-06T20:14:05,784 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/9977f7217d5a46669317b16af6a035b4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/9977f7217d5a46669317b16af6a035b4 2023-09-06T20:14:05,785 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/4e22c69bb7fe41babbc91b5c00660619 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/4e22c69bb7fe41babbc91b5c00660619 2023-09-06T20:14:05,786 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/185bbf9fb325422ab5e56a4c9904dfa7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/185bbf9fb325422ab5e56a4c9904dfa7 2023-09-06T20:14:05,787 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/61d8abe5ccfd4af394a1ed8fb64f0a1a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/61d8abe5ccfd4af394a1ed8fb64f0a1a 2023-09-06T20:14:05,788 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0013d9daef4d49a29acf38f78f5dd7a2 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0013d9daef4d49a29acf38f78f5dd7a2 2023-09-06T20:14:05,790 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/dc22e4c2e4fd4f318b7ad9f2f5080e7f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/dc22e4c2e4fd4f318b7ad9f2f5080e7f 2023-09-06T20:14:05,791 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/803ef76280da4b3fb6fde96a05477566 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/803ef76280da4b3fb6fde96a05477566 2023-09-06T20:14:05,792 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d931243386764028ae822b6e272e3f20 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d931243386764028ae822b6e272e3f20 2023-09-06T20:14:05,794 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/7ae48a1600464248a09d3beff30f5ef5 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/7ae48a1600464248a09d3beff30f5ef5 2023-09-06T20:14:05,795 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/7001ee4fb13047ab8edec7c2d51c6715 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/7001ee4fb13047ab8edec7c2d51c6715 2023-09-06T20:14:05,796 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/70e36316f3c84caab5e5128a4192f915 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/70e36316f3c84caab5e5128a4192f915 2023-09-06T20:14:05,797 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/258de79a5fc8459daedfc0e3d4db9efa to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/258de79a5fc8459daedfc0e3d4db9efa 2023-09-06T20:14:05,798 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/434e2e38a55e4fbe80031b5562f8bc56 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/434e2e38a55e4fbe80031b5562f8bc56 2023-09-06T20:14:05,800 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/a5da57ff07e1458cae5f3fe546f3483c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/a5da57ff07e1458cae5f3fe546f3483c 2023-09-06T20:14:05,801 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/79a6adcba06441fca93a771003490e8b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/79a6adcba06441fca93a771003490e8b 2023-09-06T20:14:05,803 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/56a40ab8ec4447269501ce3964553bb6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/56a40ab8ec4447269501ce3964553bb6 2023-09-06T20:14:05,804 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b915b6978c1b40fcbb385ca3baae4cc4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/b915b6978c1b40fcbb385ca3baae4cc4 2023-09-06T20:14:05,806 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/e545c21e23e8444f805867afb93f1da2 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/e545c21e23e8444f805867afb93f1da2 2023-09-06T20:14:05,807 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/214e3692091040658d21c57f4ac164f8 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/214e3692091040658d21c57f4ac164f8 2023-09-06T20:14:05,809 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/375259bb852344908484384a0a2a4888 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/375259bb852344908484384a0a2a4888 2023-09-06T20:14:05,810 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/83cc07450b994b7a87642390901b9897 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/83cc07450b994b7a87642390901b9897 2023-09-06T20:14:05,811 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/fdd0a209f2984bc3850a7f80ca299e8e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/fdd0a209f2984bc3850a7f80ca299e8e 2023-09-06T20:14:05,813 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/4045171879314bdb870b75bc9205333b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/4045171879314bdb870b75bc9205333b 2023-09-06T20:14:05,814 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d0fce4b1be0f4239a3d652b4128541a6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d0fce4b1be0f4239a3d652b4128541a6 2023-09-06T20:14:05,815 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/3174088b15504357a5c06d698343980b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/3174088b15504357a5c06d698343980b 2023-09-06T20:14:05,817 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/f990f5bf9f6d419ba3aca9aefb8eabfb to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/f990f5bf9f6d419ba3aca9aefb8eabfb 2023-09-06T20:14:05,818 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/f84d75c9f9b94f6292c775256dbe3848 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/f84d75c9f9b94f6292c775256dbe3848 2023-09-06T20:14:05,819 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/88160ceb30d541b2becb2b2e6a2af52d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/88160ceb30d541b2becb2b2e6a2af52d 2023-09-06T20:14:05,820 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/fb418dd4f023490193c26d9207401d37 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/fb418dd4f023490193c26d9207401d37 2023-09-06T20:14:05,822 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0a24ffebf24f4f7198b8d0a7b80db0d6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0a24ffebf24f4f7198b8d0a7b80db0d6 2023-09-06T20:14:05,824 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/49c372ac5e294dbcbc6143d7b23447f2 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/49c372ac5e294dbcbc6143d7b23447f2 2023-09-06T20:14:05,825 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/51ea84c3062d473db034648d5cc9f1c2 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/51ea84c3062d473db034648d5cc9f1c2 2023-09-06T20:14:05,828 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d120dc0fc60b4bbca21dc4c60ade4628, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3d5e625628264082b0ed42b3a3fc82d3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/15dbdd8c79da4cfdb396aa9099d05077, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b2b0f2c4955042a6a5bd64c6ea6dc36f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d687fa405c1a4304a7948e13556cf1e9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/adaca99d26424096bc5f54516ffd1f3d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/c125f6851bcb475fb058277cfdd13659, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/f2b1010641a94035a1e9b582fb48d607, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/055dcc36f1bf4a098b83812ac123c66c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/2da321e705fd4cb1af49b1a7c2d07deb, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/44786d4c24804e3da0a523bc557c145d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/1b277f6fc4c64ce89219f7901f8cfbe1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/58a10953dfdf4471bf5ea8163ca820e5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9d1844be2c894bcaa978ec00f765ec58, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3174afc262bc408ba16f7552a2fc86c9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a5a549cf92154d52bf3828fe31bf8998, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a780c939b1494108b7a3b2d3c391558d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0c42b50036c1493f983c8b8a390bd533, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/7cd1309a98e54412b168009c517f96ee, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b0f747af2e3c4b07b289c7b5e9682592, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/38c540a71ee74bd0a9c5304aeab82d47, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0d72023208bd42dab13e06aebe2f9f7f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a9f35bb2a4b04786ba4c1258a0c776d0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/c150c1e4e7be465d8c75d9f9c8dc6085, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9e229d3dcf7d43028cab50e5af4613b3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/4ca2359f34fa4b9baab4f4d8c9aec792, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6ff1ba35251046d883c7b3bb8961f2eb, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d599fa6b6d434d128121cd68110c73de, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3e5dfc01a0fe45e4bc3bdac775054482, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/310a78ec80fd4f139b75f98a9ef6cfdb, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b8ed8a45e03d4196b430622a85c6bcbf, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b6a7c58e47c44224b222c5547ac431df, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/28760f2b3e5b46f1bbfa2b3272e7683f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/01e2200249744ca4a047f26824ae87c6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6ef21b673c3545ee96acb63cc5aa2bbc, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0f2df07bcd5c498a83ae9b8d8ea08e3a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/8b28ad9a428647df8dc61b0d86aefc78, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/bb64bae323ed41f89fc25faa8cc2952d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/5c1d3908e3f349ceabc3f57badd077d7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9096946f28484d14905221253e98ae37, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/91ded994d92b4af9915efb216bb6010e] to archive 2023-09-06T20:14:05,829 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:14:05,831 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d120dc0fc60b4bbca21dc4c60ade4628 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d120dc0fc60b4bbca21dc4c60ade4628 2023-09-06T20:14:05,832 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3d5e625628264082b0ed42b3a3fc82d3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3d5e625628264082b0ed42b3a3fc82d3 2023-09-06T20:14:05,833 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/15dbdd8c79da4cfdb396aa9099d05077 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/15dbdd8c79da4cfdb396aa9099d05077 2023-09-06T20:14:05,835 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b2b0f2c4955042a6a5bd64c6ea6dc36f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b2b0f2c4955042a6a5bd64c6ea6dc36f 2023-09-06T20:14:05,836 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d687fa405c1a4304a7948e13556cf1e9 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d687fa405c1a4304a7948e13556cf1e9 2023-09-06T20:14:05,837 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/adaca99d26424096bc5f54516ffd1f3d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/adaca99d26424096bc5f54516ffd1f3d 2023-09-06T20:14:05,838 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/c125f6851bcb475fb058277cfdd13659 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/c125f6851bcb475fb058277cfdd13659 2023-09-06T20:14:05,839 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/f2b1010641a94035a1e9b582fb48d607 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/f2b1010641a94035a1e9b582fb48d607 2023-09-06T20:14:05,841 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/055dcc36f1bf4a098b83812ac123c66c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/055dcc36f1bf4a098b83812ac123c66c 2023-09-06T20:14:05,842 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/2da321e705fd4cb1af49b1a7c2d07deb to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/2da321e705fd4cb1af49b1a7c2d07deb 2023-09-06T20:14:05,843 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/44786d4c24804e3da0a523bc557c145d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/44786d4c24804e3da0a523bc557c145d 2023-09-06T20:14:05,844 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/1b277f6fc4c64ce89219f7901f8cfbe1 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/1b277f6fc4c64ce89219f7901f8cfbe1 2023-09-06T20:14:05,845 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/58a10953dfdf4471bf5ea8163ca820e5 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/58a10953dfdf4471bf5ea8163ca820e5 2023-09-06T20:14:05,847 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9d1844be2c894bcaa978ec00f765ec58 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9d1844be2c894bcaa978ec00f765ec58 2023-09-06T20:14:05,848 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3174afc262bc408ba16f7552a2fc86c9 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3174afc262bc408ba16f7552a2fc86c9 2023-09-06T20:14:05,849 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a5a549cf92154d52bf3828fe31bf8998 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a5a549cf92154d52bf3828fe31bf8998 2023-09-06T20:14:05,851 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a780c939b1494108b7a3b2d3c391558d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a780c939b1494108b7a3b2d3c391558d 2023-09-06T20:14:05,852 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0c42b50036c1493f983c8b8a390bd533 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0c42b50036c1493f983c8b8a390bd533 2023-09-06T20:14:05,853 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/7cd1309a98e54412b168009c517f96ee to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/7cd1309a98e54412b168009c517f96ee 2023-09-06T20:14:05,854 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b0f747af2e3c4b07b289c7b5e9682592 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b0f747af2e3c4b07b289c7b5e9682592 2023-09-06T20:14:05,855 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/38c540a71ee74bd0a9c5304aeab82d47 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/38c540a71ee74bd0a9c5304aeab82d47 2023-09-06T20:14:05,856 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0d72023208bd42dab13e06aebe2f9f7f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0d72023208bd42dab13e06aebe2f9f7f 2023-09-06T20:14:05,857 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a9f35bb2a4b04786ba4c1258a0c776d0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/a9f35bb2a4b04786ba4c1258a0c776d0 2023-09-06T20:14:05,858 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/c150c1e4e7be465d8c75d9f9c8dc6085 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/c150c1e4e7be465d8c75d9f9c8dc6085 2023-09-06T20:14:05,859 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9e229d3dcf7d43028cab50e5af4613b3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9e229d3dcf7d43028cab50e5af4613b3 2023-09-06T20:14:05,860 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/4ca2359f34fa4b9baab4f4d8c9aec792 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/4ca2359f34fa4b9baab4f4d8c9aec792 2023-09-06T20:14:05,862 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6ff1ba35251046d883c7b3bb8961f2eb to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6ff1ba35251046d883c7b3bb8961f2eb 2023-09-06T20:14:05,863 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d599fa6b6d434d128121cd68110c73de to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/d599fa6b6d434d128121cd68110c73de 2023-09-06T20:14:05,864 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3e5dfc01a0fe45e4bc3bdac775054482 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/3e5dfc01a0fe45e4bc3bdac775054482 2023-09-06T20:14:05,866 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/310a78ec80fd4f139b75f98a9ef6cfdb to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/310a78ec80fd4f139b75f98a9ef6cfdb 2023-09-06T20:14:05,867 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b8ed8a45e03d4196b430622a85c6bcbf to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b8ed8a45e03d4196b430622a85c6bcbf 2023-09-06T20:14:05,868 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b6a7c58e47c44224b222c5547ac431df to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/b6a7c58e47c44224b222c5547ac431df 2023-09-06T20:14:05,869 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/28760f2b3e5b46f1bbfa2b3272e7683f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/28760f2b3e5b46f1bbfa2b3272e7683f 2023-09-06T20:14:05,870 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/01e2200249744ca4a047f26824ae87c6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/01e2200249744ca4a047f26824ae87c6 2023-09-06T20:14:05,871 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6ef21b673c3545ee96acb63cc5aa2bbc to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6ef21b673c3545ee96acb63cc5aa2bbc 2023-09-06T20:14:05,872 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0f2df07bcd5c498a83ae9b8d8ea08e3a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/0f2df07bcd5c498a83ae9b8d8ea08e3a 2023-09-06T20:14:05,873 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/8b28ad9a428647df8dc61b0d86aefc78 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/8b28ad9a428647df8dc61b0d86aefc78 2023-09-06T20:14:05,875 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/bb64bae323ed41f89fc25faa8cc2952d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/bb64bae323ed41f89fc25faa8cc2952d 2023-09-06T20:14:05,876 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/5c1d3908e3f349ceabc3f57badd077d7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/5c1d3908e3f349ceabc3f57badd077d7 2023-09-06T20:14:05,877 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9096946f28484d14905221253e98ae37 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/9096946f28484d14905221253e98ae37 2023-09-06T20:14:05,878 DEBUG [StoreCloser-TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/91ded994d92b4af9915efb216bb6010e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/91ded994d92b4af9915efb216bb6010e 2023-09-06T20:14:05,883 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/recovered.edits/616.seqid, newMaxSeqId=616, maxSeqId=1 2023-09-06T20:14:05,884 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5. 2023-09-06T20:14:05,884 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1622): Region close journal for a659f678dfa70a1da8b63a91043e10d5: 2023-09-06T20:14:05,886 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(163): Closed a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:14:05,886 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=a659f678dfa70a1da8b63a91043e10d5, regionState=CLOSED 2023-09-06T20:14:05,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=32, resume processing ppid=31 2023-09-06T20:14:05,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, ppid=31, state=SUCCESS; CloseRegionProcedure a659f678dfa70a1da8b63a91043e10d5, server=jenkins-hbase3.apache.org,33275,1694031217268 in 1.2600 sec 2023-09-06T20:14:05,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-09-06T20:14:05,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=a659f678dfa70a1da8b63a91043e10d5, UNASSIGN in 1.2660 sec 2023-09-06T20:14:05,891 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031245891"}]},"ts":"1694031245891"} 2023-09-06T20:14:05,893 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2023-09-06T20:14:05,894 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2023-09-06T20:14:05,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.3650 sec 2023-09-06T20:14:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-06T20:14:06,644 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 30 completed 2023-09-06T20:14:06,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete TestAcidGuarantees 2023-09-06T20:14:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:06,657 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:06,658 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=33 2023-09-06T20:14:06,661 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:14:06,666 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/recovered.edits] 2023-09-06T20:14:06,670 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/52a7abef277a42d992b9bcb658e33b67 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/52a7abef277a42d992b9bcb658e33b67 2023-09-06T20:14:06,671 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/863794d8e3474d688a6d7295b8f97b38 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/863794d8e3474d688a6d7295b8f97b38 2023-09-06T20:14:06,673 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/f908f8c1cf3748808589a92595d560eb to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/A/f908f8c1cf3748808589a92595d560eb 2023-09-06T20:14:06,676 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0789f02b14bb4a59a90a94fb9faff6c5 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/0789f02b14bb4a59a90a94fb9faff6c5 2023-09-06T20:14:06,678 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/6fba37b1a20440498c1126da3250e350 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/6fba37b1a20440498c1126da3250e350 2023-09-06T20:14:06,679 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d3e79742d10346eb8d618fa25439104a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/B/d3e79742d10346eb8d618fa25439104a 2023-09-06T20:14:06,682 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/51d1a687fbd94e069a630f3dc172e89e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/51d1a687fbd94e069a630f3dc172e89e 2023-09-06T20:14:06,684 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6dbb0866af554992ba5b1608ccfe25eb to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/6dbb0866af554992ba5b1608ccfe25eb 2023-09-06T20:14:06,685 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/85c3e97d55104857abcc793af44eb6d6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/C/85c3e97d55104857abcc793af44eb6d6 2023-09-06T20:14:06,689 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/recovered.edits/616.seqid to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5/recovered.edits/616.seqid 2023-09-06T20:14:06,690 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/a659f678dfa70a1da8b63a91043e10d5 2023-09-06T20:14:06,690 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2023-09-06T20:14:06,696 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:06,718 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2023-09-06T20:14:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=33 2023-09-06T20:14:06,770 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2023-09-06T20:14:06,772 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:06,772 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2023-09-06T20:14:06,772 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694031246772"}]},"ts":"9223372036854775807"} 2023-09-06T20:14:06,775 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:14:06,775 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => a659f678dfa70a1da8b63a91043e10d5, NAME => 'TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:14:06,776 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2023-09-06T20:14:06,776 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694031246776"}]},"ts":"9223372036854775807"} 2023-09-06T20:14:06,778 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2023-09-06T20:14:06,780 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:06,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 131 msec 2023-09-06T20:14:06,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2023-09-06T20:14:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=33 2023-09-06T20:14:06,962 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 33 completed 2023-09-06T20:14:06,981 INFO [Listener at localhost/35961 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testMixedAtomicity Thread=223 (was 204) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1414633511_18 at /127.0.0.1:33872 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1414633511_18 at /127.0.0.1:33712 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS-EventLoopGroup-1-3 org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:209) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:202) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:306) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:363) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2779af75-shared-pool-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer for 'HBase' metrics system java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS:0;jenkins-hbase3:33275-shortCompactions-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2779af75-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1414633511_18 at /127.0.0.1:33726 [Waiting for operation #11] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RSProcedureDispatcher-pool-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2779af75-shared-pool-5 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2779af75-shared-pool-4 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1414633511_18 at /127.0.0.1:33818 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=652 (was 478) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=518 (was 480) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 175), AvailableMemoryMB=5139 (was 5706) 2023-09-06T20:14:06,992 INFO [Listener at localhost/35961 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testMobMixedAtomicity Thread=223, OpenFileDescriptor=652, MaxFileDescriptor=60000, SystemLoadAverage=518, ProcessCount=170, AvailableMemoryMB=5138 2023-09-06T20:14:06,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-06T20:14:06,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:14:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:06,997 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:14:06,998 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:14:06,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2023-09-06T20:14:06,998 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:14:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:14:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:14:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:14:07,406 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236 2023-09-06T20:14:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:14:07,813 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:14:07,813 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1668): Closing 05a9474df993f762d37e597f1b39ed3e, disabling compactions & flushes 2023-09-06T20:14:07,813 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:07,813 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:07,813 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. after waiting 0 ms 2023-09-06T20:14:07,813 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:07,813 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:07,813 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1622): Region close journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:07,814 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:14:07,815 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1694031247814"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694031247814"}]},"ts":"1694031247814"} 2023-09-06T20:14:07,816 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:14:07,817 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:14:07,817 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031247817"}]},"ts":"1694031247817"} 2023-09-06T20:14:07,818 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2023-09-06T20:14:07,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=05a9474df993f762d37e597f1b39ed3e, ASSIGN}] 2023-09-06T20:14:07,824 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=05a9474df993f762d37e597f1b39ed3e, ASSIGN 2023-09-06T20:14:07,825 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=05a9474df993f762d37e597f1b39ed3e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33275,1694031217268; forceNewPlan=false, retain=false 2023-09-06T20:14:07,976 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=05a9474df993f762d37e597f1b39ed3e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:07,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE; OpenRegionProcedure 05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:14:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:14:08,133 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:08,133 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7254): Opening region: {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:14:08,133 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:08,133 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:14:08,133 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7296): checking encryption for 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:08,133 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7299): checking classloading for 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:08,135 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:08,136 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:14:08,136 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05a9474df993f762d37e597f1b39ed3e columnFamilyName A 2023-09-06T20:14:08,136 DEBUG [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:14:08,137 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.HStore(324): Store=05a9474df993f762d37e597f1b39ed3e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:14:08,137 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:08,138 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:14:08,139 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05a9474df993f762d37e597f1b39ed3e columnFamilyName B 2023-09-06T20:14:08,140 DEBUG [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:14:08,140 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.HStore(324): Store=05a9474df993f762d37e597f1b39ed3e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:14:08,140 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:08,141 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:14:08,141 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05a9474df993f762d37e597f1b39ed3e columnFamilyName C 2023-09-06T20:14:08,141 DEBUG [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:14:08,142 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.HStore(324): Store=05a9474df993f762d37e597f1b39ed3e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:14:08,142 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:08,143 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:08,143 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:08,144 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-06T20:14:08,146 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1076): writing seq id for 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:08,148 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:14:08,148 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): Opened 05a9474df993f762d37e597f1b39ed3e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62744790, jitterRate=-0.06502977013587952}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-06T20:14:08,149 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(992): Region open journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:08,150 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., pid=36, masterSystemTime=1694031248129 2023-09-06T20:14:08,151 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:08,151 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:08,152 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=05a9474df993f762d37e597f1b39ed3e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:08,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=36, resume processing ppid=35 2023-09-06T20:14:08,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, ppid=35, state=SUCCESS; OpenRegionProcedure 05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 in 175 msec 2023-09-06T20:14:08,156 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-09-06T20:14:08,156 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=05a9474df993f762d37e597f1b39ed3e, ASSIGN in 331 msec 2023-09-06T20:14:08,156 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:14:08,157 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031248156"}]},"ts":"1694031248156"} 2023-09-06T20:14:08,158 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2023-09-06T20:14:08,162 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:14:08,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.1670 sec 2023-09-06T20:14:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:14:09,105 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 34 completed 2023-09-06T20:14:09,107 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6c3ba4b6 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:09,112 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@571c864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:09,114 DEBUG [Listener at localhost/35961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:09,117 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55172, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:09,119 DEBUG [Listener at localhost/35961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:14:09,121 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42508, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:14:09,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-06T20:14:09,128 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$13(2700): Client=jenkins//172.31.12.81 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:14:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:09,552 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.963 2023-09-06T20:14:09,552 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.999 2023-09-06T20:14:09,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2023-09-06T20:14:09,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=05a9474df993f762d37e597f1b39ed3e, REOPEN/MOVE}] 2023-09-06T20:14:09,571 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=05a9474df993f762d37e597f1b39ed3e, REOPEN/MOVE 2023-09-06T20:14:09,572 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=05a9474df993f762d37e597f1b39ed3e, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:09,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; CloseRegionProcedure 05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:14:09,726 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(119): Close 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:09,726 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1668): Closing 05a9474df993f762d37e597f1b39ed3e, disabling compactions & flushes 2023-09-06T20:14:09,726 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:09,726 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:09,726 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. after waiting 0 ms 2023-09-06T20:14:09,726 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:09,731 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:14:09,731 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:09,731 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1622): Region close journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:09,731 WARN [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3735): Not adding moved region record: 05a9474df993f762d37e597f1b39ed3e to self. 2023-09-06T20:14:09,733 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(163): Closed 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:09,733 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=05a9474df993f762d37e597f1b39ed3e, regionState=CLOSED 2023-09-06T20:14:09,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-09-06T20:14:09,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; CloseRegionProcedure 05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 in 161 msec 2023-09-06T20:14:09,737 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=05a9474df993f762d37e597f1b39ed3e, REOPEN/MOVE; state=CLOSED, location=jenkins-hbase3.apache.org,33275,1694031217268; forceNewPlan=false, retain=true 2023-09-06T20:14:09,762 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-06T20:14:09,764 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42510, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-06T20:14:09,887 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=05a9474df993f762d37e597f1b39ed3e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:09,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE; OpenRegionProcedure 05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:14:10,044 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:10,044 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7254): Opening region: {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:14:10,044 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:10,044 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:14:10,044 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7296): checking encryption for 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:10,044 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7299): checking classloading for 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:10,047 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:10,048 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:14:10,055 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05a9474df993f762d37e597f1b39ed3e columnFamilyName A 2023-09-06T20:14:10,057 DEBUG [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:14:10,057 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.HStore(324): Store=05a9474df993f762d37e597f1b39ed3e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:14:10,058 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:10,058 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:14:10,059 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05a9474df993f762d37e597f1b39ed3e columnFamilyName B 2023-09-06T20:14:10,059 DEBUG [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:14:10,060 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.HStore(324): Store=05a9474df993f762d37e597f1b39ed3e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:14:10,060 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:10,060 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:14:10,061 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05a9474df993f762d37e597f1b39ed3e columnFamilyName C 2023-09-06T20:14:10,061 DEBUG [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:14:10,061 INFO [StoreOpener-05a9474df993f762d37e597f1b39ed3e-1 {}] regionserver.HStore(324): Store=05a9474df993f762d37e597f1b39ed3e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:14:10,061 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:10,062 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:10,063 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:10,065 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-06T20:14:10,066 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1076): writing seq id for 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:10,067 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): Opened 05a9474df993f762d37e597f1b39ed3e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63949331, jitterRate=-0.04708071053028107}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-06T20:14:10,068 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(992): Region open journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:10,069 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., pid=41, masterSystemTime=1694031250041 2023-09-06T20:14:10,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:10,071 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:10,071 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=05a9474df993f762d37e597f1b39ed3e, regionState=OPEN, openSeqNum=5, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=39 2023-09-06T20:14:10,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=39, state=SUCCESS; OpenRegionProcedure 05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 in 183 msec 2023-09-06T20:14:10,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=39, resume processing ppid=38 2023-09-06T20:14:10,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, ppid=38, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=05a9474df993f762d37e597f1b39ed3e, REOPEN/MOVE in 504 msec 2023-09-06T20:14:10,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=38, resume processing ppid=37 2023-09-06T20:14:10,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, ppid=37, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 522 msec 2023-09-06T20:14:10,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 949 msec 2023-09-06T20:14:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-09-06T20:14:10,094 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x773a3d1e to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:10,099 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd9f0b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:10,101 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x37152f12 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:10,105 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750b4578, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:10,106 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6da744d5 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:10,112 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a8fc9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:10,114 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1e0a1074 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:10,121 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30300762, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:10,122 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x54a7d44c to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:10,128 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ae0e71c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:10,129 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5cc3635d to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:10,135 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4186a7de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:10,136 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x64362822 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:10,146 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23287672, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:10,147 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2cda7fd4 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:10,151 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79783555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:10,152 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6a78a12d to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:10,155 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a34769, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:10,161 DEBUG [hconnection-0x2a5fcce6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:10,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2023-09-06T20:14:10,168 DEBUG [hconnection-0x34e4bea-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:10,168 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55174, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:10,168 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:10,169 DEBUG [hconnection-0x763bb04f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:14:10,170 DEBUG [hconnection-0x4b1fa1d3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:10,170 DEBUG [hconnection-0x19237ece-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:10,170 DEBUG [hconnection-0x30e5087b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:10,171 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37386, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:10,171 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37422, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:10,176 DEBUG [hconnection-0x3474985b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:10,176 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37396, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:10,176 DEBUG [hconnection-0x4a966aac-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:10,176 DEBUG [hconnection-0x7064b7bb-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:10,178 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37448, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:10,179 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37382, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:10,179 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37430, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:10,179 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:10,180 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37406, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:10,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:10,180 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37438, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:10,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:14:10,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:10,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:10,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:10,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:10,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:10,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:10,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309066a6a03e0fe1a489c98e2c3f6920184b1_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031250202/Put/seqid=0 2023-09-06T20:14:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:14:10,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031310278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031310279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031310282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031310286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031310287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2023-09-06T20:14:10,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:10,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:10,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:10,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(53): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:10,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:10,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031310384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031310384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031310386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031310389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031310389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:14:10,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2023-09-06T20:14:10,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:10,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:10,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:10,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(53): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:10,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:10,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031310593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031310593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031310593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031310593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031310593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2023-09-06T20:14:10,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:10,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:10,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:10,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(53): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:10,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:10,714 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309066a6a03e0fe1a489c98e2c3f6920184b1_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309066a6a03e0fe1a489c98e2c3f6920184b1_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:10,715 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/f22c95452e1741e68e14372644832446, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:10,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/f22c95452e1741e68e14372644832446 is 175, key is test_row_0/A:col10/1694031250202/Put/seqid=0 2023-09-06T20:14:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:14:10,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2023-09-06T20:14:10,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:10,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:10,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:10,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(53): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:10,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:10,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031310896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031310897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031310898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031310898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031310902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:10,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2023-09-06T20:14:10,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:10,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:10,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:10,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(53): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:10,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:11,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2023-09-06T20:14:11,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:11,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:11,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:11,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(53): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:11,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:11,163 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/f22c95452e1741e68e14372644832446 2023-09-06T20:14:11,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/106b73fbee5e450e8ccec05309f97344 is 50, key is test_row_0/B:col10/1694031250202/Put/seqid=0 2023-09-06T20:14:11,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/106b73fbee5e450e8ccec05309f97344 2023-09-06T20:14:11,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/78d8b51d6de94b72a2cd506b915aecc4 is 50, key is test_row_0/C:col10/1694031250202/Put/seqid=0 2023-09-06T20:14:11,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2023-09-06T20:14:11,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:11,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:11,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:11,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(53): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:11,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:14:11,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031311400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:11,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031311402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:11,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031311403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:11,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031311404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:11,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031311406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:11,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2023-09-06T20:14:11,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:11,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:11,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:11,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(53): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:11,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:11,426 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-06T20:14:11,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2023-09-06T20:14:11,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:11,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:11,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:11,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(53): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:11,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:11,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/78d8b51d6de94b72a2cd506b915aecc4 2023-09-06T20:14:11,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/f22c95452e1741e68e14372644832446 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/f22c95452e1741e68e14372644832446 2023-09-06T20:14:11,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/f22c95452e1741e68e14372644832446, entries=150, sequenceid=17, filesize=30.2 K 2023-09-06T20:14:11,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/106b73fbee5e450e8ccec05309f97344 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/106b73fbee5e450e8ccec05309f97344 2023-09-06T20:14:11,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/106b73fbee5e450e8ccec05309f97344, entries=150, sequenceid=17, filesize=11.7 K 2023-09-06T20:14:11,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/78d8b51d6de94b72a2cd506b915aecc4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/78d8b51d6de94b72a2cd506b915aecc4 2023-09-06T20:14:11,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/78d8b51d6de94b72a2cd506b915aecc4, entries=150, sequenceid=17, filesize=11.7 K 2023-09-06T20:14:11,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 05a9474df993f762d37e597f1b39ed3e in 1497ms, sequenceid=17, compaction requested=false 2023-09-06T20:14:11,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:11,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2023-09-06T20:14:11,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:11,721 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-06T20:14:11,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:11,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:11,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:11,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:11,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:11,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:11,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090615d98b8006064f8c87e10cef920d4eaa_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031250278/Put/seqid=0 2023-09-06T20:14:11,770 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090615d98b8006064f8c87e10cef920d4eaa_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090615d98b8006064f8c87e10cef920d4eaa_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:11,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/edcbda4b44e842f6ad7b5477946fac8f, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:11,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/edcbda4b44e842f6ad7b5477946fac8f is 175, key is test_row_0/A:col10/1694031250278/Put/seqid=0 2023-09-06T20:14:11,782 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/edcbda4b44e842f6ad7b5477946fac8f 2023-09-06T20:14:11,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/a0547b74e1f14a2898ae944ff59d5a9f is 50, key is test_row_0/B:col10/1694031250278/Put/seqid=0 2023-09-06T20:14:11,830 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/a0547b74e1f14a2898ae944ff59d5a9f 2023-09-06T20:14:11,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/fe74dce9198a4f46a16bcf999d6176a6 is 50, key is test_row_0/C:col10/1694031250278/Put/seqid=0 2023-09-06T20:14:11,861 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/fe74dce9198a4f46a16bcf999d6176a6 2023-09-06T20:14:11,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/edcbda4b44e842f6ad7b5477946fac8f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/edcbda4b44e842f6ad7b5477946fac8f 2023-09-06T20:14:11,874 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/edcbda4b44e842f6ad7b5477946fac8f, entries=150, sequenceid=41, filesize=30.2 K 2023-09-06T20:14:11,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/a0547b74e1f14a2898ae944ff59d5a9f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/a0547b74e1f14a2898ae944ff59d5a9f 2023-09-06T20:14:11,881 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/a0547b74e1f14a2898ae944ff59d5a9f, entries=150, sequenceid=41, filesize=11.7 K 2023-09-06T20:14:11,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/fe74dce9198a4f46a16bcf999d6176a6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/fe74dce9198a4f46a16bcf999d6176a6 2023-09-06T20:14:11,888 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/fe74dce9198a4f46a16bcf999d6176a6, entries=150, sequenceid=41, filesize=11.7 K 2023-09-06T20:14:11,889 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 05a9474df993f762d37e597f1b39ed3e in 168ms, sequenceid=41, compaction requested=false 2023-09-06T20:14:11,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:11,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:11,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=43 2023-09-06T20:14:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=43 2023-09-06T20:14:11,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-09-06T20:14:11,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7110 sec 2023-09-06T20:14:11,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.7330 sec 2023-09-06T20:14:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:14:12,279 INFO [Thread-855 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 42 completed 2023-09-06T20:14:12,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2023-09-06T20:14:12,284 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:14:12,285 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:12,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:14:12,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2023-09-06T20:14:12,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:12,440 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2023-09-06T20:14:12,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:12,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:12,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:12,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:12,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:12,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:12,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906e930b740e0fa4460a5d99e37e6da0c13_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031252438/Put/seqid=0 2023-09-06T20:14:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:12,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:12,494 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906e930b740e0fa4460a5d99e37e6da0c13_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906e930b740e0fa4460a5d99e37e6da0c13_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:12,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/a5904564161f4e76adc1584e055fedf5, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:12,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/a5904564161f4e76adc1584e055fedf5 is 175, key is test_row_0/A:col10/1694031252438/Put/seqid=0 2023-09-06T20:14:12,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031312510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031312514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031312517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031312517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,524 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/a5904564161f4e76adc1584e055fedf5 2023-09-06T20:14:12,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031312524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/45761bc470b04adbb81e05f5782ffc5e is 50, key is test_row_0/B:col10/1694031252438/Put/seqid=0 2023-09-06T20:14:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:14:12,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031312618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031312620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031312624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031312625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031312626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031312820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031312822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031312827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031312828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:12,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031312829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:12,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:14:12,980 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/45761bc470b04adbb81e05f5782ffc5e 2023-09-06T20:14:12,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/d4944075f43e4790a5effd4fd70d0033 is 50, key is test_row_0/C:col10/1694031252438/Put/seqid=0 2023-09-06T20:14:13,037 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/d4944075f43e4790a5effd4fd70d0033 2023-09-06T20:14:13,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/a5904564161f4e76adc1584e055fedf5 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/a5904564161f4e76adc1584e055fedf5 2023-09-06T20:14:13,052 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/a5904564161f4e76adc1584e055fedf5, entries=250, sequenceid=49, filesize=47.0 K 2023-09-06T20:14:13,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/45761bc470b04adbb81e05f5782ffc5e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/45761bc470b04adbb81e05f5782ffc5e 2023-09-06T20:14:13,060 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/45761bc470b04adbb81e05f5782ffc5e, entries=100, sequenceid=49, filesize=9.4 K 2023-09-06T20:14:13,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/d4944075f43e4790a5effd4fd70d0033 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d4944075f43e4790a5effd4fd70d0033 2023-09-06T20:14:13,072 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d4944075f43e4790a5effd4fd70d0033, entries=100, sequenceid=49, filesize=9.4 K 2023-09-06T20:14:13,073 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3022): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=181.14 KB/185490 for 05a9474df993f762d37e597f1b39ed3e in 634ms, sequenceid=49, compaction requested=true 2023-09-06T20:14:13,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:13,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:13,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=45 2023-09-06T20:14:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=45 2023-09-06T20:14:13,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-09-06T20:14:13,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 790 msec 2023-09-06T20:14:13,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 796 msec 2023-09-06T20:14:13,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:13,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2023-09-06T20:14:13,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:13,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:13,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:13,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:13,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:13,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:13,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031313130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031313132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031313134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031313134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031313135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309065ff2cc70e0724e0690705d4dbe6a6476_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031252514/Put/seqid=0 2023-09-06T20:14:13,198 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309065ff2cc70e0724e0690705d4dbe6a6476_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309065ff2cc70e0724e0690705d4dbe6a6476_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:13,199 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/080e7540ba7244f294fa4d31aa1d589c, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:13,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/080e7540ba7244f294fa4d31aa1d589c is 175, key is test_row_0/A:col10/1694031252514/Put/seqid=0 2023-09-06T20:14:13,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031313234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031313237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031313237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,241 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=64.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/080e7540ba7244f294fa4d31aa1d589c 2023-09-06T20:14:13,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/85d6a3d1f31a44698262f045d0af11ca is 50, key is test_row_0/B:col10/1694031252514/Put/seqid=0 2023-09-06T20:14:13,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/85d6a3d1f31a44698262f045d0af11ca 2023-09-06T20:14:13,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/bde37ef3020c42f58cac2a5012301c08 is 50, key is test_row_0/C:col10/1694031252514/Put/seqid=0 2023-09-06T20:14:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:14:13,388 INFO [Thread-855 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 44 completed 2023-09-06T20:14:13,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2023-09-06T20:14:13,393 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-09-06T20:14:13,393 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:13,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:13,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031313438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031313439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031313441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-09-06T20:14:13,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2023-09-06T20:14:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:13,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(53): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:13,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031313640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031313642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-09-06T20:14:13,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2023-09-06T20:14:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:13,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:13,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:13,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(53): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:13,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:13,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031313743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:13,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031313743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031313744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:13,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/bde37ef3020c42f58cac2a5012301c08 2023-09-06T20:14:13,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/080e7540ba7244f294fa4d31aa1d589c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/080e7540ba7244f294fa4d31aa1d589c 2023-09-06T20:14:13,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/080e7540ba7244f294fa4d31aa1d589c, entries=150, sequenceid=81, filesize=30.2 K 2023-09-06T20:14:13,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/85d6a3d1f31a44698262f045d0af11ca as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/85d6a3d1f31a44698262f045d0af11ca 2023-09-06T20:14:13,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/85d6a3d1f31a44698262f045d0af11ca, entries=150, sequenceid=81, filesize=11.7 K 2023-09-06T20:14:13,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/bde37ef3020c42f58cac2a5012301c08 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/bde37ef3020c42f58cac2a5012301c08 2023-09-06T20:14:13,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/bde37ef3020c42f58cac2a5012301c08, entries=150, sequenceid=81, filesize=11.7 K 2023-09-06T20:14:13,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for 05a9474df993f762d37e597f1b39ed3e in 655ms, sequenceid=81, compaction requested=true 2023-09-06T20:14:13,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:13,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:13,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:13,786 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:13,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:13,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:13,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:13,786 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:13,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:13,788 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 140932 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:13,788 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/A is initiating minor compaction (all files) 2023-09-06T20:14:13,788 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/A in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:13,788 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/f22c95452e1741e68e14372644832446, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/edcbda4b44e842f6ad7b5477946fac8f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/a5904564161f4e76adc1584e055fedf5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/080e7540ba7244f294fa4d31aa1d589c] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=137.6 K 2023-09-06T20:14:13,788 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:13,788 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/f22c95452e1741e68e14372644832446, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/edcbda4b44e842f6ad7b5477946fac8f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/a5904564161f4e76adc1584e055fedf5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/080e7540ba7244f294fa4d31aa1d589c] 2023-09-06T20:14:13,789 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting f22c95452e1741e68e14372644832446, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1694031250202 2023-09-06T20:14:13,789 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:13,789 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/B is initiating minor compaction (all files) 2023-09-06T20:14:13,789 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/B in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:13,789 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/106b73fbee5e450e8ccec05309f97344, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/a0547b74e1f14a2898ae944ff59d5a9f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/45761bc470b04adbb81e05f5782ffc5e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/85d6a3d1f31a44698262f045d0af11ca] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=44.5 K 2023-09-06T20:14:13,789 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting edcbda4b44e842f6ad7b5477946fac8f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1694031250278 2023-09-06T20:14:13,790 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 106b73fbee5e450e8ccec05309f97344, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1694031250202 2023-09-06T20:14:13,790 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting a5904564161f4e76adc1584e055fedf5, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1694031252433 2023-09-06T20:14:13,790 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting a0547b74e1f14a2898ae944ff59d5a9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1694031250278 2023-09-06T20:14:13,791 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 080e7540ba7244f294fa4d31aa1d589c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1694031252507 2023-09-06T20:14:13,791 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 45761bc470b04adbb81e05f5782ffc5e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1694031252437 2023-09-06T20:14:13,792 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 85d6a3d1f31a44698262f045d0af11ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1694031252507 2023-09-06T20:14:13,806 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:13,811 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#B#compaction#146 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:13,812 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/94a2493a0ce344ea9118433236712888 is 50, key is test_row_0/B:col10/1694031252514/Put/seqid=0 2023-09-06T20:14:13,822 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309067e00fd34933b46569175366ed9300d4a_05a9474df993f762d37e597f1b39ed3e store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:13,825 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309067e00fd34933b46569175366ed9300d4a_05a9474df993f762d37e597f1b39ed3e, store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:13,825 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309067e00fd34933b46569175366ed9300d4a_05a9474df993f762d37e597f1b39ed3e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:13,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2023-09-06T20:14:13,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:13,855 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2023-09-06T20:14:13,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:13,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:13,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:13,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:13,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:13,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:13,892 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/94a2493a0ce344ea9118433236712888 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/94a2493a0ce344ea9118433236712888 2023-09-06T20:14:13,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906fd9e6b6cdd6b452f935359cef684706f_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_1/A:col10/1694031253131/Put/seqid=0 2023-09-06T20:14:13,900 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/B of 05a9474df993f762d37e597f1b39ed3e into 94a2493a0ce344ea9118433236712888(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:13,900 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:13,900 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/B, priority=12, startTime=1694031253786; duration=0sec 2023-09-06T20:14:13,900 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:13,900 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:B 2023-09-06T20:14:13,900 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:13,902 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:13,902 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/C is initiating minor compaction (all files) 2023-09-06T20:14:13,902 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/C in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:13,902 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/78d8b51d6de94b72a2cd506b915aecc4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/fe74dce9198a4f46a16bcf999d6176a6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d4944075f43e4790a5effd4fd70d0033, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/bde37ef3020c42f58cac2a5012301c08] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=44.5 K 2023-09-06T20:14:13,903 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 78d8b51d6de94b72a2cd506b915aecc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1694031250202 2023-09-06T20:14:13,903 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting fe74dce9198a4f46a16bcf999d6176a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1694031250278 2023-09-06T20:14:13,904 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting d4944075f43e4790a5effd4fd70d0033, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1694031252437 2023-09-06T20:14:13,905 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting bde37ef3020c42f58cac2a5012301c08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1694031252507 2023-09-06T20:14:13,917 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#C#compaction#148 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:13,918 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/f4c750df0b7944f5b599f834a6565654 is 50, key is test_row_0/C:col10/1694031252514/Put/seqid=0 2023-09-06T20:14:13,918 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906fd9e6b6cdd6b452f935359cef684706f_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906fd9e6b6cdd6b452f935359cef684706f_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/7d2533ddf5644d7f89e39f593fb7491f, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:13,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/7d2533ddf5644d7f89e39f593fb7491f is 175, key is test_row_1/A:col10/1694031253131/Put/seqid=0 2023-09-06T20:14:13,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-09-06T20:14:14,274 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#A#compaction#145 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:14,278 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/c4e0486c1e4a4558aa867d4a9d028da0 is 175, key is test_row_0/A:col10/1694031252514/Put/seqid=0 2023-09-06T20:14:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:14,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:14,299 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/c4e0486c1e4a4558aa867d4a9d028da0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/c4e0486c1e4a4558aa867d4a9d028da0 2023-09-06T20:14:14,309 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/A of 05a9474df993f762d37e597f1b39ed3e into c4e0486c1e4a4558aa867d4a9d028da0(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:14,309 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:14,309 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/A, priority=12, startTime=1694031253786; duration=0sec 2023-09-06T20:14:14,309 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:14,309 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:A 2023-09-06T20:14:14,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031314315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:14,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031314316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:14,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031314317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:14,350 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/f4c750df0b7944f5b599f834a6565654 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f4c750df0b7944f5b599f834a6565654 2023-09-06T20:14:14,357 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/C of 05a9474df993f762d37e597f1b39ed3e into f4c750df0b7944f5b599f834a6565654(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:14,357 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:14,357 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/C, priority=12, startTime=1694031253786; duration=0sec 2023-09-06T20:14:14,357 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:14,357 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:C 2023-09-06T20:14:14,359 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/7d2533ddf5644d7f89e39f593fb7491f 2023-09-06T20:14:14,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/ed06e68dea4046ff9b88cc4ec8e4f6a0 is 50, key is test_row_1/B:col10/1694031253131/Put/seqid=0 2023-09-06T20:14:14,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031314419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:14,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031314421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:14,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031314421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-09-06T20:14:14,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031314621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:14,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031314623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:14,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031314624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:14,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031314648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:14,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031314651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:14,797 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/ed06e68dea4046ff9b88cc4ec8e4f6a0 2023-09-06T20:14:14,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/f61968c64db14e2c97fb2663567a2799 is 50, key is test_row_1/C:col10/1694031253131/Put/seqid=0 2023-09-06T20:14:14,812 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/f61968c64db14e2c97fb2663567a2799 2023-09-06T20:14:14,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/7d2533ddf5644d7f89e39f593fb7491f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/7d2533ddf5644d7f89e39f593fb7491f 2023-09-06T20:14:14,822 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/7d2533ddf5644d7f89e39f593fb7491f, entries=50, sequenceid=85, filesize=13.4 K 2023-09-06T20:14:14,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/ed06e68dea4046ff9b88cc4ec8e4f6a0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/ed06e68dea4046ff9b88cc4ec8e4f6a0 2023-09-06T20:14:14,828 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/ed06e68dea4046ff9b88cc4ec8e4f6a0, entries=50, sequenceid=85, filesize=7.1 K 2023-09-06T20:14:14,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/f61968c64db14e2c97fb2663567a2799 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f61968c64db14e2c97fb2663567a2799 2023-09-06T20:14:14,834 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f61968c64db14e2c97fb2663567a2799, entries=50, sequenceid=85, filesize=7.1 K 2023-09-06T20:14:14,834 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3022): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=194.56 KB/199230 for 05a9474df993f762d37e597f1b39ed3e in 979ms, sequenceid=85, compaction requested=false 2023-09-06T20:14:14,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:14,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:14,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=47 2023-09-06T20:14:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=47 2023-09-06T20:14:14,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=47, resume processing ppid=46 2023-09-06T20:14:14,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, ppid=46, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4430 sec 2023-09-06T20:14:14,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4480 sec 2023-09-06T20:14:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:14,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=207.98 KB heapSize=545.67 KB 2023-09-06T20:14:14,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:14,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031314930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:14,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:14,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:14,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:14,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:14,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:14,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031314931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:14,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031314931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:14,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906ee8ce9c5e03442c682383051459e1267_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031254315/Put/seqid=0 2023-09-06T20:14:14,954 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906ee8ce9c5e03442c682383051459e1267_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906ee8ce9c5e03442c682383051459e1267_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:14,955 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/08cd7f180448459f9082203e9053e04a, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:14,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/08cd7f180448459f9082203e9053e04a is 175, key is test_row_0/A:col10/1694031254315/Put/seqid=0 2023-09-06T20:14:14,962 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=69.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/08cd7f180448459f9082203e9053e04a 2023-09-06T20:14:14,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/16916d2e938f428a8cf1bd947094b271 is 50, key is test_row_0/B:col10/1694031254315/Put/seqid=0 2023-09-06T20:14:14,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/16916d2e938f428a8cf1bd947094b271 2023-09-06T20:14:14,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/01643590d0af462fb050a11527852039 is 50, key is test_row_0/C:col10/1694031254315/Put/seqid=0 2023-09-06T20:14:15,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/01643590d0af462fb050a11527852039 2023-09-06T20:14:15,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/08cd7f180448459f9082203e9053e04a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/08cd7f180448459f9082203e9053e04a 2023-09-06T20:14:15,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/08cd7f180448459f9082203e9053e04a, entries=150, sequenceid=122, filesize=30.2 K 2023-09-06T20:14:15,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/16916d2e938f428a8cf1bd947094b271 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/16916d2e938f428a8cf1bd947094b271 2023-09-06T20:14:15,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/16916d2e938f428a8cf1bd947094b271, entries=150, sequenceid=122, filesize=11.7 K 2023-09-06T20:14:15,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/01643590d0af462fb050a11527852039 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/01643590d0af462fb050a11527852039 2023-09-06T20:14:15,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031315033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:15,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/01643590d0af462fb050a11527852039, entries=150, sequenceid=122, filesize=11.7 K 2023-09-06T20:14:15,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~207.98 KB/212970, heapSize ~545.63 KB/558720, currentSize=0 B/0 for 05a9474df993f762d37e597f1b39ed3e in 142ms, sequenceid=122, compaction requested=true 2023-09-06T20:14:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:15,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:15,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:15,074 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:15,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:15,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:15,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:15,074 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:15,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:15,075 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 75762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:15,075 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/A is initiating minor compaction (all files) 2023-09-06T20:14:15,075 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/A in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:15,075 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31404 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:15,075 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/B is initiating minor compaction (all files) 2023-09-06T20:14:15,075 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/c4e0486c1e4a4558aa867d4a9d028da0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/7d2533ddf5644d7f89e39f593fb7491f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/08cd7f180448459f9082203e9053e04a] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=74.0 K 2023-09-06T20:14:15,075 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:15,075 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/B in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:15,075 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/c4e0486c1e4a4558aa867d4a9d028da0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/7d2533ddf5644d7f89e39f593fb7491f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/08cd7f180448459f9082203e9053e04a] 2023-09-06T20:14:15,075 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/94a2493a0ce344ea9118433236712888, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/ed06e68dea4046ff9b88cc4ec8e4f6a0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/16916d2e938f428a8cf1bd947094b271] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=30.7 K 2023-09-06T20:14:15,076 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting c4e0486c1e4a4558aa867d4a9d028da0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1694031252507 2023-09-06T20:14:15,076 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 94a2493a0ce344ea9118433236712888, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1694031252507 2023-09-06T20:14:15,076 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7d2533ddf5644d7f89e39f593fb7491f, keycount=50, bloomtype=ROW, size=13.4 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1694031253131 2023-09-06T20:14:15,078 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting ed06e68dea4046ff9b88cc4ec8e4f6a0, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1694031253131 2023-09-06T20:14:15,078 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 08cd7f180448459f9082203e9053e04a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1694031254311 2023-09-06T20:14:15,080 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 16916d2e938f428a8cf1bd947094b271, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1694031254311 2023-09-06T20:14:15,089 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#B#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:15,090 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/6b74138bb5714ad79d97c33c56ee70b7 is 50, key is test_row_0/B:col10/1694031254315/Put/seqid=0 2023-09-06T20:14:15,091 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:15,099 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230906cdf951297e184946999ad3d535529e00_05a9474df993f762d37e597f1b39ed3e store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:15,100 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230906cdf951297e184946999ad3d535529e00_05a9474df993f762d37e597f1b39ed3e, store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:15,101 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906cdf951297e184946999ad3d535529e00_05a9474df993f762d37e597f1b39ed3e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:15,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:14:15,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:15,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:15,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:15,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:15,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:15,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:15,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c64592adebdb4dae8fb07e344ceb43d1_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031255158/Put/seqid=0 2023-09-06T20:14:15,245 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c64592adebdb4dae8fb07e344ceb43d1_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c64592adebdb4dae8fb07e344ceb43d1_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:15,246 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/6454e248f41e4aba8bc770c10da3af09, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:15,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/6454e248f41e4aba8bc770c10da3af09 is 175, key is test_row_0/A:col10/1694031255158/Put/seqid=0 2023-09-06T20:14:15,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031315250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:15,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031315272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:15,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031315354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:15,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031315437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-09-06T20:14:15,499 INFO [Thread-855 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 46 completed 2023-09-06T20:14:15,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2023-09-06T20:14:15,504 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:15,504 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:15,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:14:15,517 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/6b74138bb5714ad79d97c33c56ee70b7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/6b74138bb5714ad79d97c33c56ee70b7 2023-09-06T20:14:15,517 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#A#compaction#155 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:15,518 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/d7a70d40c93048c5b52f5a5093fe4794 is 175, key is test_row_0/A:col10/1694031254315/Put/seqid=0 2023-09-06T20:14:15,527 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/B of 05a9474df993f762d37e597f1b39ed3e into 6b74138bb5714ad79d97c33c56ee70b7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:15,527 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:15,527 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/B, priority=13, startTime=1694031255074; duration=0sec 2023-09-06T20:14:15,527 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:15,527 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:B 2023-09-06T20:14:15,528 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:15,529 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31404 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:15,529 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/C is initiating minor compaction (all files) 2023-09-06T20:14:15,530 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/C in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:15,530 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f4c750df0b7944f5b599f834a6565654, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f61968c64db14e2c97fb2663567a2799, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/01643590d0af462fb050a11527852039] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=30.7 K 2023-09-06T20:14:15,530 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting f4c750df0b7944f5b599f834a6565654, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1694031252507 2023-09-06T20:14:15,531 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting f61968c64db14e2c97fb2663567a2799, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1694031253131 2023-09-06T20:14:15,531 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 01643590d0af462fb050a11527852039, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1694031254311 2023-09-06T20:14:15,538 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#C#compaction#157 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:15,539 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/83fb82e52c92410aaef2f99867b6c64c is 50, key is test_row_0/C:col10/1694031254315/Put/seqid=0 2023-09-06T20:14:15,557 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/83fb82e52c92410aaef2f99867b6c64c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/83fb82e52c92410aaef2f99867b6c64c 2023-09-06T20:14:15,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031315558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:15,564 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/C of 05a9474df993f762d37e597f1b39ed3e into 83fb82e52c92410aaef2f99867b6c64c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:15,564 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:15,565 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/C, priority=13, startTime=1694031255074; duration=0sec 2023-09-06T20:14:15,565 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:15,565 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:C 2023-09-06T20:14:15,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031315578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:14:15,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2023-09-06T20:14:15,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:15,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:15,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:15,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(53): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:15,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:15,665 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/6454e248f41e4aba8bc770c10da3af09 2023-09-06T20:14:15,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/5481b594f5d048ea82cb8b84f8ead42a is 50, key is test_row_0/B:col10/1694031255158/Put/seqid=0 2023-09-06T20:14:15,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/5481b594f5d048ea82cb8b84f8ead42a 2023-09-06T20:14:15,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/672a776a69e84959b0faf914b1259a23 is 50, key is test_row_0/C:col10/1694031255158/Put/seqid=0 2023-09-06T20:14:15,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:14:15,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2023-09-06T20:14:15,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:15,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:15,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:15,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(53): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:15,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:15,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031315860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:15,950 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/d7a70d40c93048c5b52f5a5093fe4794 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/d7a70d40c93048c5b52f5a5093fe4794 2023-09-06T20:14:15,955 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/A of 05a9474df993f762d37e597f1b39ed3e into d7a70d40c93048c5b52f5a5093fe4794(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:15,955 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:15,955 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/A, priority=13, startTime=1694031255073; duration=0sec 2023-09-06T20:14:15,955 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:15,955 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:A 2023-09-06T20:14:15,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2023-09-06T20:14:15,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:15,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:15,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:15,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(53): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:16,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031316085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:16,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/672a776a69e84959b0faf914b1259a23 2023-09-06T20:14:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:14:16,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/6454e248f41e4aba8bc770c10da3af09 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/6454e248f41e4aba8bc770c10da3af09 2023-09-06T20:14:16,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2023-09-06T20:14:16,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:16,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:16,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:16,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(53): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:16,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:16,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/6454e248f41e4aba8bc770c10da3af09, entries=200, sequenceid=133, filesize=38.8 K 2023-09-06T20:14:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:16,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/5481b594f5d048ea82cb8b84f8ead42a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/5481b594f5d048ea82cb8b84f8ead42a 2023-09-06T20:14:16,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/5481b594f5d048ea82cb8b84f8ead42a, entries=150, sequenceid=133, filesize=11.8 K 2023-09-06T20:14:16,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/672a776a69e84959b0faf914b1259a23 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/672a776a69e84959b0faf914b1259a23 2023-09-06T20:14:16,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/672a776a69e84959b0faf914b1259a23, entries=150, sequenceid=133, filesize=11.8 K 2023-09-06T20:14:16,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 05a9474df993f762d37e597f1b39ed3e in 975ms, sequenceid=133, compaction requested=false 2023-09-06T20:14:16,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:16,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2023-09-06T20:14:16,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:16,273 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-06T20:14:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:16,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309069c38c0fc51614352af481f32257d5e4a_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031255228/Put/seqid=0 2023-09-06T20:14:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:16,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:16,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031316387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:16,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031316450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:16,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031316488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:14:16,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031316652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:16,655 DEBUG [Thread-847 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4136 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., hostname=jenkins-hbase3.apache.org,33275,1694031217268, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:130) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-06T20:14:16,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:16,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031316670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:16,675 DEBUG [Thread-851 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4150 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., hostname=jenkins-hbase3.apache.org,33275,1694031217268, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:130) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-06T20:14:16,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:16,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031316692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:16,714 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309069c38c0fc51614352af481f32257d5e4a_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309069c38c0fc51614352af481f32257d5e4a_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:16,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/884d93b87f9c408fb277698e1b75337a, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:16,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/884d93b87f9c408fb277698e1b75337a is 175, key is test_row_0/A:col10/1694031255228/Put/seqid=0 2023-09-06T20:14:16,740 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/884d93b87f9c408fb277698e1b75337a 2023-09-06T20:14:16,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/0ba3a42582ee4f9d808caa20a7d9e92c is 50, key is test_row_0/B:col10/1694031255228/Put/seqid=0 2023-09-06T20:14:16,765 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/0ba3a42582ee4f9d808caa20a7d9e92c 2023-09-06T20:14:16,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/19983c582907468fada196b7ff5120c0 is 50, key is test_row_0/C:col10/1694031255228/Put/seqid=0 2023-09-06T20:14:16,782 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/19983c582907468fada196b7ff5120c0 2023-09-06T20:14:16,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/884d93b87f9c408fb277698e1b75337a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/884d93b87f9c408fb277698e1b75337a 2023-09-06T20:14:16,794 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/884d93b87f9c408fb277698e1b75337a, entries=150, sequenceid=161, filesize=30.4 K 2023-09-06T20:14:16,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/0ba3a42582ee4f9d808caa20a7d9e92c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/0ba3a42582ee4f9d808caa20a7d9e92c 2023-09-06T20:14:16,804 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/0ba3a42582ee4f9d808caa20a7d9e92c, entries=150, sequenceid=161, filesize=11.8 K 2023-09-06T20:14:16,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/19983c582907468fada196b7ff5120c0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/19983c582907468fada196b7ff5120c0 2023-09-06T20:14:16,823 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/19983c582907468fada196b7ff5120c0, entries=150, sequenceid=161, filesize=11.8 K 2023-09-06T20:14:16,825 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 05a9474df993f762d37e597f1b39ed3e in 552ms, sequenceid=161, compaction requested=true 2023-09-06T20:14:16,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:16,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:16,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=49 2023-09-06T20:14:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=49 2023-09-06T20:14:16,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-09-06T20:14:16,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3220 sec 2023-09-06T20:14:16,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.3270 sec 2023-09-06T20:14:16,896 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2023-09-06T20:14:16,896 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2023-09-06T20:14:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:17,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:14:17,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:17,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:17,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:17,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:17,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:17,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:17,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906cfea31ac0d7f4612b2c34ebbcbf8ba33_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_1/A:col10/1694031257003/Put/seqid=0 2023-09-06T20:14:17,078 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906cfea31ac0d7f4612b2c34ebbcbf8ba33_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906cfea31ac0d7f4612b2c34ebbcbf8ba33_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:17,079 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/a2777eaa81ce4567b55ec48a88b33092, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:17,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/a2777eaa81ce4567b55ec48a88b33092 is 175, key is test_row_1/A:col10/1694031257003/Put/seqid=0 2023-09-06T20:14:17,093 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/a2777eaa81ce4567b55ec48a88b33092 2023-09-06T20:14:17,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/55f6979fe92b41608f62709ce6425d2b is 50, key is test_row_1/B:col10/1694031257003/Put/seqid=0 2023-09-06T20:14:17,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031317114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:17,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031317116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:17,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031317218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:17,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031317220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:17,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031317422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:17,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031317427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:17,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/55f6979fe92b41608f62709ce6425d2b 2023-09-06T20:14:17,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/33ec0ebf76144b129b331be1ebf2185b is 50, key is test_row_1/C:col10/1694031257003/Put/seqid=0 2023-09-06T20:14:17,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:14:17,612 INFO [Thread-855 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 48 completed 2023-09-06T20:14:17,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:17,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2023-09-06T20:14:17,617 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:14:17,618 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:17,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:14:17,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031317727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:17,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031317730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:17,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2023-09-06T20:14:17,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:17,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:17,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:17,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(53): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:17,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:14:17,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2023-09-06T20:14:17,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:17,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:17,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:17,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(53): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:17,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:18,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/33ec0ebf76144b129b331be1ebf2185b 2023-09-06T20:14:18,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/a2777eaa81ce4567b55ec48a88b33092 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/a2777eaa81ce4567b55ec48a88b33092 2023-09-06T20:14:18,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/a2777eaa81ce4567b55ec48a88b33092, entries=150, sequenceid=173, filesize=30.4 K 2023-09-06T20:14:18,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/55f6979fe92b41608f62709ce6425d2b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/55f6979fe92b41608f62709ce6425d2b 2023-09-06T20:14:18,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/55f6979fe92b41608f62709ce6425d2b, entries=100, sequenceid=173, filesize=9.5 K 2023-09-06T20:14:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/33ec0ebf76144b129b331be1ebf2185b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/33ec0ebf76144b129b331be1ebf2185b 2023-09-06T20:14:18,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/33ec0ebf76144b129b331be1ebf2185b, entries=100, sequenceid=173, filesize=9.5 K 2023-09-06T20:14:18,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 05a9474df993f762d37e597f1b39ed3e in 1027ms, sequenceid=173, compaction requested=true 2023-09-06T20:14:18,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:18,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:18,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:18,032 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:18,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:18,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:18,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:18,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-06T20:14:18,033 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:18,035 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46182 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:18,036 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/B is initiating minor compaction (all files) 2023-09-06T20:14:18,036 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/B in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:18,036 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/6b74138bb5714ad79d97c33c56ee70b7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/5481b594f5d048ea82cb8b84f8ead42a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/0ba3a42582ee4f9d808caa20a7d9e92c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/55f6979fe92b41608f62709ce6425d2b] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=45.1 K 2023-09-06T20:14:18,036 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133032 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:18,036 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/A is initiating minor compaction (all files) 2023-09-06T20:14:18,036 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/A in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:18,036 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/d7a70d40c93048c5b52f5a5093fe4794, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/6454e248f41e4aba8bc770c10da3af09, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/884d93b87f9c408fb277698e1b75337a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/a2777eaa81ce4567b55ec48a88b33092] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=129.9 K 2023-09-06T20:14:18,037 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:18,037 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/d7a70d40c93048c5b52f5a5093fe4794, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/6454e248f41e4aba8bc770c10da3af09, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/884d93b87f9c408fb277698e1b75337a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/a2777eaa81ce4567b55ec48a88b33092] 2023-09-06T20:14:18,038 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting d7a70d40c93048c5b52f5a5093fe4794, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1694031254311 2023-09-06T20:14:18,038 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 6b74138bb5714ad79d97c33c56ee70b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1694031254311 2023-09-06T20:14:18,039 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6454e248f41e4aba8bc770c10da3af09, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1694031255134 2023-09-06T20:14:18,039 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 5481b594f5d048ea82cb8b84f8ead42a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1694031255134 2023-09-06T20:14:18,040 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 0ba3a42582ee4f9d808caa20a7d9e92c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1694031255228 2023-09-06T20:14:18,040 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 884d93b87f9c408fb277698e1b75337a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1694031255228 2023-09-06T20:14:18,040 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 55f6979fe92b41608f62709ce6425d2b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1694031256383 2023-09-06T20:14:18,041 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting a2777eaa81ce4567b55ec48a88b33092, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1694031256383 2023-09-06T20:14:18,053 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#B#compaction#166 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:18,053 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/eb7162e990704e1b89d1147d4e85dc9c is 50, key is test_row_0/B:col10/1694031255228/Put/seqid=0 2023-09-06T20:14:18,062 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:18,079 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230906b214c0415dd644d7a3544b7bf014a3b5_05a9474df993f762d37e597f1b39ed3e store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:18,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2023-09-06T20:14:18,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:18,080 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-06T20:14:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:18,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:18,081 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230906b214c0415dd644d7a3544b7bf014a3b5_05a9474df993f762d37e597f1b39ed3e, store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:18,081 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906b214c0415dd644d7a3544b7bf014a3b5_05a9474df993f762d37e597f1b39ed3e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:18,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309064c86ced9476342a8a3cf70bdcd777f3b_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031257114/Put/seqid=0 2023-09-06T20:14:18,127 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309064c86ced9476342a8a3cf70bdcd777f3b_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309064c86ced9476342a8a3cf70bdcd777f3b_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:18,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/df8ea28dd894438588115a8f8a2a14ab, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:18,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/df8ea28dd894438588115a8f8a2a14ab is 175, key is test_row_0/A:col10/1694031257114/Put/seqid=0 2023-09-06T20:14:18,142 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/df8ea28dd894438588115a8f8a2a14ab 2023-09-06T20:14:18,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/ec697cff18d0460da8705cada7280f9d is 50, key is test_row_0/B:col10/1694031257114/Put/seqid=0 2023-09-06T20:14:18,172 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/ec697cff18d0460da8705cada7280f9d 2023-09-06T20:14:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/5e6d3b64ffbb4f3cbe57823619420dad is 50, key is test_row_0/C:col10/1694031257114/Put/seqid=0 2023-09-06T20:14:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:14:18,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:18,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031318263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:18,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031318263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:18,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031318370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:18,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031318370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:18,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031318456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:18,463 DEBUG [Thread-853 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4146 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., hostname=jenkins-hbase3.apache.org,33275,1694031217268, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:130) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-06T20:14:18,504 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#A#compaction#167 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:18,505 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/2c5dc3431a694d5cac62406054923e5f is 175, key is test_row_0/A:col10/1694031255228/Put/seqid=0 2023-09-06T20:14:18,517 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/eb7162e990704e1b89d1147d4e85dc9c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/eb7162e990704e1b89d1147d4e85dc9c 2023-09-06T20:14:18,523 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/B of 05a9474df993f762d37e597f1b39ed3e into eb7162e990704e1b89d1147d4e85dc9c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:18,523 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:18,523 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/B, priority=12, startTime=1694031258032; duration=0sec 2023-09-06T20:14:18,523 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:18,523 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:B 2023-09-06T20:14:18,523 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:18,526 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46182 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:18,526 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/C is initiating minor compaction (all files) 2023-09-06T20:14:18,526 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/C in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:18,526 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/83fb82e52c92410aaef2f99867b6c64c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/672a776a69e84959b0faf914b1259a23, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/19983c582907468fada196b7ff5120c0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/33ec0ebf76144b129b331be1ebf2185b] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=45.1 K 2023-09-06T20:14:18,527 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 83fb82e52c92410aaef2f99867b6c64c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1694031254311 2023-09-06T20:14:18,528 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 672a776a69e84959b0faf914b1259a23, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1694031255134 2023-09-06T20:14:18,528 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 19983c582907468fada196b7ff5120c0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1694031255228 2023-09-06T20:14:18,529 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 33ec0ebf76144b129b331be1ebf2185b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1694031256383 2023-09-06T20:14:18,540 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/2c5dc3431a694d5cac62406054923e5f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/2c5dc3431a694d5cac62406054923e5f 2023-09-06T20:14:18,543 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#C#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:18,544 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/53c1b79c70fa4660898e11e63da51d0f is 50, key is test_row_0/C:col10/1694031255228/Put/seqid=0 2023-09-06T20:14:18,548 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/A of 05a9474df993f762d37e597f1b39ed3e into 2c5dc3431a694d5cac62406054923e5f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:18,548 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:18,548 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/A, priority=12, startTime=1694031258032; duration=0sec 2023-09-06T20:14:18,549 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:18,549 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:A 2023-09-06T20:14:18,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031318573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:18,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031318574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:18,596 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/53c1b79c70fa4660898e11e63da51d0f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/53c1b79c70fa4660898e11e63da51d0f 2023-09-06T20:14:18,606 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/C of 05a9474df993f762d37e597f1b39ed3e into 53c1b79c70fa4660898e11e63da51d0f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:18,606 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:18,606 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/C, priority=12, startTime=1694031258032; duration=0sec 2023-09-06T20:14:18,606 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:18,606 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:C 2023-09-06T20:14:18,624 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/5e6d3b64ffbb4f3cbe57823619420dad 2023-09-06T20:14:18,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/df8ea28dd894438588115a8f8a2a14ab as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/df8ea28dd894438588115a8f8a2a14ab 2023-09-06T20:14:18,638 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/df8ea28dd894438588115a8f8a2a14ab, entries=150, sequenceid=197, filesize=30.4 K 2023-09-06T20:14:18,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/ec697cff18d0460da8705cada7280f9d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/ec697cff18d0460da8705cada7280f9d 2023-09-06T20:14:18,660 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/ec697cff18d0460da8705cada7280f9d, entries=150, sequenceid=197, filesize=11.8 K 2023-09-06T20:14:18,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/5e6d3b64ffbb4f3cbe57823619420dad as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/5e6d3b64ffbb4f3cbe57823619420dad 2023-09-06T20:14:18,666 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/5e6d3b64ffbb4f3cbe57823619420dad, entries=150, sequenceid=197, filesize=11.8 K 2023-09-06T20:14:18,667 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 05a9474df993f762d37e597f1b39ed3e in 587ms, sequenceid=197, compaction requested=false 2023-09-06T20:14:18,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:18,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:18,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2023-09-06T20:14:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=51 2023-09-06T20:14:18,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-09-06T20:14:18,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0500 sec 2023-09-06T20:14:18,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.0570 sec 2023-09-06T20:14:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:14:18,722 INFO [Thread-855 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 50 completed 2023-09-06T20:14:18,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2023-09-06T20:14:18,727 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:18,727 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:18,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:14:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:14:18,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2023-09-06T20:14:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:18,881 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:14:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:18,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309068c243d07d59a4781b6a890688b322274_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_1/A:col10/1694031258258/Put/seqid=0 2023-09-06T20:14:18,934 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309068c243d07d59a4781b6a890688b322274_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309068c243d07d59a4781b6a890688b322274_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:18,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/d525e3ec7b7a415f934596b18dfea79e, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:18,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/d525e3ec7b7a415f934596b18dfea79e is 175, key is test_row_1/A:col10/1694031258258/Put/seqid=0 2023-09-06T20:14:18,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031318944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:18,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:18,948 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/d525e3ec7b7a415f934596b18dfea79e 2023-09-06T20:14:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031318947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:18,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/f966a0a90b724f7a8a317c5a117553f8 is 50, key is test_row_1/B:col10/1694031258258/Put/seqid=0 2023-09-06T20:14:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:14:19,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031319048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:19,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031319049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:19,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031319252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:19,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031319253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:14:19,372 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/f966a0a90b724f7a8a317c5a117553f8 2023-09-06T20:14:19,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/f4272f452c5b4c9581e30fedf50ecc7c is 50, key is test_row_1/C:col10/1694031258258/Put/seqid=0 2023-09-06T20:14:19,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031319554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:19,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031319555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:19,785 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/f4272f452c5b4c9581e30fedf50ecc7c 2023-09-06T20:14:19,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/d525e3ec7b7a415f934596b18dfea79e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/d525e3ec7b7a415f934596b18dfea79e 2023-09-06T20:14:19,796 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/d525e3ec7b7a415f934596b18dfea79e, entries=100, sequenceid=212, filesize=21.9 K 2023-09-06T20:14:19,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/f966a0a90b724f7a8a317c5a117553f8 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/f966a0a90b724f7a8a317c5a117553f8 2023-09-06T20:14:19,802 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/f966a0a90b724f7a8a317c5a117553f8, entries=100, sequenceid=212, filesize=9.5 K 2023-09-06T20:14:19,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/f4272f452c5b4c9581e30fedf50ecc7c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f4272f452c5b4c9581e30fedf50ecc7c 2023-09-06T20:14:19,812 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f4272f452c5b4c9581e30fedf50ecc7c, entries=100, sequenceid=212, filesize=9.5 K 2023-09-06T20:14:19,813 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 05a9474df993f762d37e597f1b39ed3e in 932ms, sequenceid=212, compaction requested=true 2023-09-06T20:14:19,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:19,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:19,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-09-06T20:14:19,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=53 2023-09-06T20:14:19,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-09-06T20:14:19,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0870 sec 2023-09-06T20:14:19,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.0930 sec 2023-09-06T20:14:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:14:19,835 INFO [Thread-855 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 52 completed 2023-09-06T20:14:19,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2023-09-06T20:14:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-06T20:14:19,840 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:19,840 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:19,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-06T20:14:19,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2023-09-06T20:14:19,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:19,994 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-06T20:14:19,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:19,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:19,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:19,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:19,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:19,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:20,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906cdf11344ad2149b8b380d32dd119b26e_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031258938/Put/seqid=0 2023-09-06T20:14:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:20,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:20,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031320071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:20,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031320072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-06T20:14:20,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031320174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:20,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031320174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:20,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031320378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:20,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031320379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:20,413 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906cdf11344ad2149b8b380d32dd119b26e_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906cdf11344ad2149b8b380d32dd119b26e_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:20,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/1e14fdf24cfe42faa92c0cb0d2b879c4, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/1e14fdf24cfe42faa92c0cb0d2b879c4 is 175, key is test_row_0/A:col10/1694031258938/Put/seqid=0 2023-09-06T20:14:20,425 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/1e14fdf24cfe42faa92c0cb0d2b879c4 2023-09-06T20:14:20,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/8ca5ccb2db2445259c1b9060a9266558 is 50, key is test_row_0/B:col10/1694031258938/Put/seqid=0 2023-09-06T20:14:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-06T20:14:20,458 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/8ca5ccb2db2445259c1b9060a9266558 2023-09-06T20:14:20,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/54282fe9b0bc46c1bbe49dfefc604951 is 50, key is test_row_0/C:col10/1694031258938/Put/seqid=0 2023-09-06T20:14:20,504 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/54282fe9b0bc46c1bbe49dfefc604951 2023-09-06T20:14:20,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/1e14fdf24cfe42faa92c0cb0d2b879c4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/1e14fdf24cfe42faa92c0cb0d2b879c4 2023-09-06T20:14:20,516 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/1e14fdf24cfe42faa92c0cb0d2b879c4, entries=150, sequenceid=236, filesize=30.4 K 2023-09-06T20:14:20,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/8ca5ccb2db2445259c1b9060a9266558 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/8ca5ccb2db2445259c1b9060a9266558 2023-09-06T20:14:20,523 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/8ca5ccb2db2445259c1b9060a9266558, entries=150, sequenceid=236, filesize=11.8 K 2023-09-06T20:14:20,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/54282fe9b0bc46c1bbe49dfefc604951 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/54282fe9b0bc46c1bbe49dfefc604951 2023-09-06T20:14:20,530 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/54282fe9b0bc46c1bbe49dfefc604951, entries=150, sequenceid=236, filesize=11.8 K 2023-09-06T20:14:20,531 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 05a9474df993f762d37e597f1b39ed3e in 537ms, sequenceid=236, compaction requested=true 2023-09-06T20:14:20,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:20,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:20,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=55 2023-09-06T20:14:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=55 2023-09-06T20:14:20,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-09-06T20:14:20,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 692 msec 2023-09-06T20:14:20,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 697 msec 2023-09-06T20:14:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:20,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-06T20:14:20,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:20,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:20,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:20,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:20,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:20,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:20,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090630c2370529264841ab2fec5a63b66423_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031260068/Put/seqid=0 2023-09-06T20:14:20,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031320760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:20,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:20,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031320760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031320760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:20,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031320765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:20,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031320866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:20,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031320866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:20,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031320866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:20,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031320868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-06T20:14:20,944 INFO [Thread-855 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 54 completed 2023-09-06T20:14:20,945 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2023-09-06T20:14:20,948 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-06T20:14:20,949 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:20,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-06T20:14:21,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031321068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031321068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031321069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031321070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,099 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090630c2370529264841ab2fec5a63b66423_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090630c2370529264841ab2fec5a63b66423_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:21,100 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/cf188553c6f045ca91cbd25389a84c7c, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:21,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/cf188553c6f045ca91cbd25389a84c7c is 175, key is test_row_0/A:col10/1694031260068/Put/seqid=0 2023-09-06T20:14:21,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2023-09-06T20:14:21,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:21,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:21,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:21,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(53): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:21,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-06T20:14:21,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2023-09-06T20:14:21,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:21,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:21,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:21,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(53): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:21,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:21,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:21,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031321372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031321374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031321374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031321374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2023-09-06T20:14:21,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:21,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:21,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:21,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(53): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:21,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:21,520 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/cf188553c6f045ca91cbd25389a84c7c 2023-09-06T20:14:21,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/9e1bac5bd71a47a2a3fa2edccaa08746 is 50, key is test_row_0/B:col10/1694031260068/Put/seqid=0 2023-09-06T20:14:21,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/9e1bac5bd71a47a2a3fa2edccaa08746 2023-09-06T20:14:21,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/59e0b5988e204f7988661f13e5a70314 is 50, key is test_row_0/C:col10/1694031260068/Put/seqid=0 2023-09-06T20:14:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-06T20:14:21,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/59e0b5988e204f7988661f13e5a70314 2023-09-06T20:14:21,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2023-09-06T20:14:21,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:21,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:21,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:21,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(53): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:21,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:21,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/cf188553c6f045ca91cbd25389a84c7c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/cf188553c6f045ca91cbd25389a84c7c 2023-09-06T20:14:21,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/cf188553c6f045ca91cbd25389a84c7c, entries=150, sequenceid=249, filesize=30.4 K 2023-09-06T20:14:21,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/9e1bac5bd71a47a2a3fa2edccaa08746 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/9e1bac5bd71a47a2a3fa2edccaa08746 2023-09-06T20:14:21,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/9e1bac5bd71a47a2a3fa2edccaa08746, entries=150, sequenceid=249, filesize=11.8 K 2023-09-06T20:14:21,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/59e0b5988e204f7988661f13e5a70314 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/59e0b5988e204f7988661f13e5a70314 2023-09-06T20:14:21,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/59e0b5988e204f7988661f13e5a70314, entries=150, sequenceid=249, filesize=11.8 K 2023-09-06T20:14:21,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 05a9474df993f762d37e597f1b39ed3e in 917ms, sequenceid=249, compaction requested=true 2023-09-06T20:14:21,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:21,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:21,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:21,591 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2023-09-06T20:14:21,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:21,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:21,591 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2023-09-06T20:14:21,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:21,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:21,593 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 147279 starting at candidate #0 after considering 6 permutations with 6 in ratio 2023-09-06T20:14:21,593 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/A is initiating minor compaction (all files) 2023-09-06T20:14:21,593 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58652 starting at candidate #0 after considering 6 permutations with 6 in ratio 2023-09-06T20:14:21,593 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/A in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:21,593 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/B is initiating minor compaction (all files) 2023-09-06T20:14:21,593 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/B in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:21,593 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/2c5dc3431a694d5cac62406054923e5f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/df8ea28dd894438588115a8f8a2a14ab, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/d525e3ec7b7a415f934596b18dfea79e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/1e14fdf24cfe42faa92c0cb0d2b879c4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/cf188553c6f045ca91cbd25389a84c7c] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=143.8 K 2023-09-06T20:14:21,593 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/eb7162e990704e1b89d1147d4e85dc9c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/ec697cff18d0460da8705cada7280f9d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/f966a0a90b724f7a8a317c5a117553f8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/8ca5ccb2db2445259c1b9060a9266558, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/9e1bac5bd71a47a2a3fa2edccaa08746] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=57.3 K 2023-09-06T20:14:21,593 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:21,593 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/2c5dc3431a694d5cac62406054923e5f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/df8ea28dd894438588115a8f8a2a14ab, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/d525e3ec7b7a415f934596b18dfea79e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/1e14fdf24cfe42faa92c0cb0d2b879c4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/cf188553c6f045ca91cbd25389a84c7c] 2023-09-06T20:14:21,594 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2c5dc3431a694d5cac62406054923e5f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1694031255228 2023-09-06T20:14:21,594 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting eb7162e990704e1b89d1147d4e85dc9c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1694031255228 2023-09-06T20:14:21,595 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting df8ea28dd894438588115a8f8a2a14ab, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1694031257097 2023-09-06T20:14:21,595 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting d525e3ec7b7a415f934596b18dfea79e, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1694031258258 2023-09-06T20:14:21,595 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting ec697cff18d0460da8705cada7280f9d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1694031257097 2023-09-06T20:14:21,596 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1e14fdf24cfe42faa92c0cb0d2b879c4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1694031258928 2023-09-06T20:14:21,596 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting f966a0a90b724f7a8a317c5a117553f8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1694031258258 2023-09-06T20:14:21,596 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting cf188553c6f045ca91cbd25389a84c7c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1694031260068 2023-09-06T20:14:21,597 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 8ca5ccb2db2445259c1b9060a9266558, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1694031258928 2023-09-06T20:14:21,597 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 9e1bac5bd71a47a2a3fa2edccaa08746, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1694031260068 2023-09-06T20:14:21,606 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:21,614 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230906928ac70e39e7441b86f26252164c5a9d_05a9474df993f762d37e597f1b39ed3e store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:21,616 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#B#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:21,616 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230906928ac70e39e7441b86f26252164c5a9d_05a9474df993f762d37e597f1b39ed3e, store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:21,616 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/68a4962d4ae3468ea01869111f6e5ebe is 50, key is test_row_0/B:col10/1694031260068/Put/seqid=0 2023-09-06T20:14:21,617 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906928ac70e39e7441b86f26252164c5a9d_05a9474df993f762d37e597f1b39ed3e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:21,672 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/68a4962d4ae3468ea01869111f6e5ebe as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/68a4962d4ae3468ea01869111f6e5ebe 2023-09-06T20:14:21,681 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 5 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/B of 05a9474df993f762d37e597f1b39ed3e into 68a4962d4ae3468ea01869111f6e5ebe(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:21,681 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:21,681 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/B, priority=11, startTime=1694031261591; duration=0sec 2023-09-06T20:14:21,681 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:21,681 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:B 2023-09-06T20:14:21,681 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2023-09-06T20:14:21,683 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58652 starting at candidate #0 after considering 6 permutations with 6 in ratio 2023-09-06T20:14:21,684 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/C is initiating minor compaction (all files) 2023-09-06T20:14:21,684 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/C in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:21,684 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/53c1b79c70fa4660898e11e63da51d0f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/5e6d3b64ffbb4f3cbe57823619420dad, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f4272f452c5b4c9581e30fedf50ecc7c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/54282fe9b0bc46c1bbe49dfefc604951, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/59e0b5988e204f7988661f13e5a70314] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=57.3 K 2023-09-06T20:14:21,684 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 53c1b79c70fa4660898e11e63da51d0f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1694031255228 2023-09-06T20:14:21,684 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 5e6d3b64ffbb4f3cbe57823619420dad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1694031257097 2023-09-06T20:14:21,685 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting f4272f452c5b4c9581e30fedf50ecc7c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1694031258258 2023-09-06T20:14:21,685 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 54282fe9b0bc46c1bbe49dfefc604951, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1694031258928 2023-09-06T20:14:21,685 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 59e0b5988e204f7988661f13e5a70314, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1694031260068 2023-09-06T20:14:21,712 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#C#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:21,713 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/947142655eba46d28f346cff822b26fd is 50, key is test_row_0/C:col10/1694031260068/Put/seqid=0 2023-09-06T20:14:21,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2023-09-06T20:14:21,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:21,717 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-06T20:14:21,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:21,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:21,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:21,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:21,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:21,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:21,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090643f0efca52714d29bab993d3eb512485_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031260764/Put/seqid=0 2023-09-06T20:14:21,768 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/947142655eba46d28f346cff822b26fd as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/947142655eba46d28f346cff822b26fd 2023-09-06T20:14:21,780 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 5 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/C of 05a9474df993f762d37e597f1b39ed3e into 947142655eba46d28f346cff822b26fd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:21,780 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:21,780 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/C, priority=11, startTime=1694031261591; duration=0sec 2023-09-06T20:14:21,780 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:21,780 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:C 2023-09-06T20:14:21,800 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090643f0efca52714d29bab993d3eb512485_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090643f0efca52714d29bab993d3eb512485_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:21,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/4541243c942a4524870d06729d4b5b02, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:21,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/4541243c942a4524870d06729d4b5b02 is 175, key is test_row_0/A:col10/1694031260764/Put/seqid=0 2023-09-06T20:14:21,827 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/4541243c942a4524870d06729d4b5b02 2023-09-06T20:14:21,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/787f4687c6f24b419b947fc397db782b is 50, key is test_row_0/B:col10/1694031260764/Put/seqid=0 2023-09-06T20:14:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:21,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:21,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031321889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031321890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031321891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031321891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,965 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-09-06T20:14:21,965 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-09-06T20:14:21,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031321996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031321997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031321997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031321998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-06T20:14:22,078 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#A#compaction#181 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:22,078 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/b2a42e3cd8e442418b46bb824c021225 is 175, key is test_row_0/A:col10/1694031260068/Put/seqid=0 2023-09-06T20:14:22,087 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/b2a42e3cd8e442418b46bb824c021225 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/b2a42e3cd8e442418b46bb824c021225 2023-09-06T20:14:22,094 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 5 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/A of 05a9474df993f762d37e597f1b39ed3e into b2a42e3cd8e442418b46bb824c021225(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:22,094 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:22,094 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/A, priority=11, startTime=1694031261591; duration=0sec 2023-09-06T20:14:22,094 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:22,094 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:A 2023-09-06T20:14:22,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031322199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031322200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031322200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031322201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,296 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/787f4687c6f24b419b947fc397db782b 2023-09-06T20:14:22,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/1a8ccffcae234f2f880ef87cbe3626e5 is 50, key is test_row_0/C:col10/1694031260764/Put/seqid=0 2023-09-06T20:14:22,308 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/1a8ccffcae234f2f880ef87cbe3626e5 2023-09-06T20:14:22,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/4541243c942a4524870d06729d4b5b02 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/4541243c942a4524870d06729d4b5b02 2023-09-06T20:14:22,326 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/4541243c942a4524870d06729d4b5b02, entries=150, sequenceid=273, filesize=30.5 K 2023-09-06T20:14:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/787f4687c6f24b419b947fc397db782b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/787f4687c6f24b419b947fc397db782b 2023-09-06T20:14:22,332 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/787f4687c6f24b419b947fc397db782b, entries=150, sequenceid=273, filesize=12.0 K 2023-09-06T20:14:22,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/1a8ccffcae234f2f880ef87cbe3626e5 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/1a8ccffcae234f2f880ef87cbe3626e5 2023-09-06T20:14:22,337 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/1a8ccffcae234f2f880ef87cbe3626e5, entries=150, sequenceid=273, filesize=12.0 K 2023-09-06T20:14:22,338 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 05a9474df993f762d37e597f1b39ed3e in 621ms, sequenceid=273, compaction requested=false 2023-09-06T20:14:22,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:22,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:22,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=57 2023-09-06T20:14:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=57 2023-09-06T20:14:22,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=57, resume processing ppid=56 2023-09-06T20:14:22,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, ppid=56, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3910 sec 2023-09-06T20:14:22,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.3970 sec 2023-09-06T20:14:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:22,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-06T20:14:22,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:22,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:22,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:22,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:22,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:22,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:22,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090643362edc89ef498fb6aede7a10488383_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031262484/Put/seqid=0 2023-09-06T20:14:22,507 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090643362edc89ef498fb6aede7a10488383_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090643362edc89ef498fb6aede7a10488383_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:22,508 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/9d92854eb936450f9233bd36da58b42d, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:22,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/9d92854eb936450f9233bd36da58b42d is 175, key is test_row_0/A:col10/1694031262484/Put/seqid=0 2023-09-06T20:14:22,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031322512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031322513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031322516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031322517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031322522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031322619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031322619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031322623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031322623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031322624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031322822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031322823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031322825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031322825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031322827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:22,916 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/9d92854eb936450f9233bd36da58b42d 2023-09-06T20:14:22,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/829465f90e2a4cffad250109c42db172 is 50, key is test_row_0/B:col10/1694031262484/Put/seqid=0 2023-09-06T20:14:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-06T20:14:23,053 INFO [Thread-855 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 56 completed 2023-09-06T20:14:23,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2023-09-06T20:14:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-06T20:14:23,057 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:23,057 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:23,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:23,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031323127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031323128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031323130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031323130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031323130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-06T20:14:23,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2023-09-06T20:14:23,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:23,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:23,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:23,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(53): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:23,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:23,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/829465f90e2a4cffad250109c42db172 2023-09-06T20:14:23,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/a1f08d9a777b4d0fbd0359c7e9cf2963 is 50, key is test_row_0/C:col10/1694031262484/Put/seqid=0 2023-09-06T20:14:23,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/a1f08d9a777b4d0fbd0359c7e9cf2963 2023-09-06T20:14:23,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/9d92854eb936450f9233bd36da58b42d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9d92854eb936450f9233bd36da58b42d 2023-09-06T20:14:23,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9d92854eb936450f9233bd36da58b42d, entries=150, sequenceid=291, filesize=30.5 K 2023-09-06T20:14:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/829465f90e2a4cffad250109c42db172 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/829465f90e2a4cffad250109c42db172 2023-09-06T20:14:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-06T20:14:23,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/829465f90e2a4cffad250109c42db172, entries=150, sequenceid=291, filesize=12.0 K 2023-09-06T20:14:23,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2023-09-06T20:14:23,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:23,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:23,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:23,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(53): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:23,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:23,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/a1f08d9a777b4d0fbd0359c7e9cf2963 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/a1f08d9a777b4d0fbd0359c7e9cf2963 2023-09-06T20:14:23,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/a1f08d9a777b4d0fbd0359c7e9cf2963, entries=150, sequenceid=291, filesize=12.0 K 2023-09-06T20:14:23,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 05a9474df993f762d37e597f1b39ed3e in 897ms, sequenceid=291, compaction requested=true 2023-09-06T20:14:23,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:23,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:23,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:23,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:23,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:23,381 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:23,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:23,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-06T20:14:23,381 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:23,382 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94110 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:23,383 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/A is initiating minor compaction (all files) 2023-09-06T20:14:23,383 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37248 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:23,383 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/A in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:23,383 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/B is initiating minor compaction (all files) 2023-09-06T20:14:23,383 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/B in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:23,383 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/b2a42e3cd8e442418b46bb824c021225, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/4541243c942a4524870d06729d4b5b02, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9d92854eb936450f9233bd36da58b42d] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=91.9 K 2023-09-06T20:14:23,383 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/68a4962d4ae3468ea01869111f6e5ebe, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/787f4687c6f24b419b947fc397db782b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/829465f90e2a4cffad250109c42db172] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=36.4 K 2023-09-06T20:14:23,383 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:23,383 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/b2a42e3cd8e442418b46bb824c021225, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/4541243c942a4524870d06729d4b5b02, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9d92854eb936450f9233bd36da58b42d] 2023-09-06T20:14:23,383 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b2a42e3cd8e442418b46bb824c021225, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1694031260068 2023-09-06T20:14:23,384 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 68a4962d4ae3468ea01869111f6e5ebe, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1694031260068 2023-09-06T20:14:23,384 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 4541243c942a4524870d06729d4b5b02, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1694031260718 2023-09-06T20:14:23,385 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 787f4687c6f24b419b947fc397db782b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1694031260718 2023-09-06T20:14:23,385 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 9d92854eb936450f9233bd36da58b42d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1694031261886 2023-09-06T20:14:23,386 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 829465f90e2a4cffad250109c42db172, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1694031261886 2023-09-06T20:14:23,396 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:23,399 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#B#compaction#191 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:23,399 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230906c9a2d2af58dd458f92c33a6e6fe00879_05a9474df993f762d37e597f1b39ed3e store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:23,399 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/47055ab476684b72b15ea8f69c5638e4 is 50, key is test_row_0/B:col10/1694031262484/Put/seqid=0 2023-09-06T20:14:23,400 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230906c9a2d2af58dd458f92c33a6e6fe00879_05a9474df993f762d37e597f1b39ed3e, store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:23,400 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c9a2d2af58dd458f92c33a6e6fe00879_05a9474df993f762d37e597f1b39ed3e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:23,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2023-09-06T20:14:23,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:23,518 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2023-09-06T20:14:23,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:23,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:23,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:23,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:23,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:23,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:23,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c5ffc7a8d9d74b07b84b188738ad68c9_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031262516/Put/seqid=0 2023-09-06T20:14:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:23,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-06T20:14:23,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031323641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031323642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031323670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031323670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031323670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031323771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031323771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031323775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031323775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031323776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,816 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#A#compaction#190 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:23,817 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/74044d2852bf4cbc860df1c97cc73974 is 175, key is test_row_0/A:col10/1694031262484/Put/seqid=0 2023-09-06T20:14:23,833 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/47055ab476684b72b15ea8f69c5638e4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/47055ab476684b72b15ea8f69c5638e4 2023-09-06T20:14:23,840 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/74044d2852bf4cbc860df1c97cc73974 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/74044d2852bf4cbc860df1c97cc73974 2023-09-06T20:14:23,840 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/B of 05a9474df993f762d37e597f1b39ed3e into 47055ab476684b72b15ea8f69c5638e4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:23,840 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:23,840 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/B, priority=13, startTime=1694031263381; duration=0sec 2023-09-06T20:14:23,841 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:23,841 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:B 2023-09-06T20:14:23,841 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:23,843 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37248 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:23,843 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/C is initiating minor compaction (all files) 2023-09-06T20:14:23,843 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/C in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:23,843 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/947142655eba46d28f346cff822b26fd, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/1a8ccffcae234f2f880ef87cbe3626e5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/a1f08d9a777b4d0fbd0359c7e9cf2963] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=36.4 K 2023-09-06T20:14:23,843 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 947142655eba46d28f346cff822b26fd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1694031260068 2023-09-06T20:14:23,844 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1a8ccffcae234f2f880ef87cbe3626e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1694031260718 2023-09-06T20:14:23,844 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting a1f08d9a777b4d0fbd0359c7e9cf2963, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1694031261886 2023-09-06T20:14:23,845 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/A of 05a9474df993f762d37e597f1b39ed3e into 74044d2852bf4cbc860df1c97cc73974(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:23,845 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:23,845 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/A, priority=13, startTime=1694031263381; duration=0sec 2023-09-06T20:14:23,846 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:23,846 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:A 2023-09-06T20:14:23,859 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#C#compaction#193 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:23,859 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/e56dfd4493fb4801b778f21adbb339f0 is 50, key is test_row_0/C:col10/1694031262484/Put/seqid=0 2023-09-06T20:14:23,871 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/e56dfd4493fb4801b778f21adbb339f0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/e56dfd4493fb4801b778f21adbb339f0 2023-09-06T20:14:23,879 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/C of 05a9474df993f762d37e597f1b39ed3e into e56dfd4493fb4801b778f21adbb339f0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:23,879 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:23,879 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/C, priority=13, startTime=1694031263381; duration=0sec 2023-09-06T20:14:23,879 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:23,879 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:C 2023-09-06T20:14:23,975 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c5ffc7a8d9d74b07b84b188738ad68c9_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c5ffc7a8d9d74b07b84b188738ad68c9_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:23,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/9adffabe33b44f01a661e2ce0f362079, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:23,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/9adffabe33b44f01a661e2ce0f362079 is 175, key is test_row_0/A:col10/1694031262516/Put/seqid=0 2023-09-06T20:14:23,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031323974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031323974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031323978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031323979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:23,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031323982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-06T20:14:24,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031324280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:24,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031324281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:24,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031324282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:24,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031324282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:24,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031324284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:24,381 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/9adffabe33b44f01a661e2ce0f362079 2023-09-06T20:14:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/e0a4bed26dd6492bbb27674c711c896c is 50, key is test_row_0/B:col10/1694031262516/Put/seqid=0 2023-09-06T20:14:24,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031324783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:24,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031324786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:24,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031324786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:24,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031324787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:24,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031324788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:24,796 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/e0a4bed26dd6492bbb27674c711c896c 2023-09-06T20:14:24,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/475bcf83842d4ff5a66059ae345500c7 is 50, key is test_row_0/C:col10/1694031262516/Put/seqid=0 2023-09-06T20:14:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-06T20:14:25,212 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/475bcf83842d4ff5a66059ae345500c7 2023-09-06T20:14:25,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/9adffabe33b44f01a661e2ce0f362079 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9adffabe33b44f01a661e2ce0f362079 2023-09-06T20:14:25,222 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9adffabe33b44f01a661e2ce0f362079, entries=150, sequenceid=312, filesize=30.5 K 2023-09-06T20:14:25,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/e0a4bed26dd6492bbb27674c711c896c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/e0a4bed26dd6492bbb27674c711c896c 2023-09-06T20:14:25,226 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/e0a4bed26dd6492bbb27674c711c896c, entries=150, sequenceid=312, filesize=12.0 K 2023-09-06T20:14:25,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/475bcf83842d4ff5a66059ae345500c7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/475bcf83842d4ff5a66059ae345500c7 2023-09-06T20:14:25,232 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/475bcf83842d4ff5a66059ae345500c7, entries=150, sequenceid=312, filesize=12.0 K 2023-09-06T20:14:25,233 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3022): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 05a9474df993f762d37e597f1b39ed3e in 1715ms, sequenceid=312, compaction requested=false 2023-09-06T20:14:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=59 2023-09-06T20:14:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=59 2023-09-06T20:14:25,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=59, resume processing ppid=58 2023-09-06T20:14:25,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, ppid=58, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1760 sec 2023-09-06T20:14:25,238 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.1820 sec 2023-09-06T20:14:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:25,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2023-09-06T20:14:25,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:25,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:25,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:25,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:25,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:25,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:25,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309065cec533a7cb440e6b3278e9887eb642a_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031263646/Put/seqid=0 2023-09-06T20:14:25,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031325813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:25,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031325813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:25,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031325814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:25,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:25,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031325814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031325814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:25,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031325917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:25,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031325917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:25,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031325919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:25,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031325926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:25,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031325926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031326122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031326122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031326123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031326138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031326138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,216 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309065cec533a7cb440e6b3278e9887eb642a_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309065cec533a7cb440e6b3278e9887eb642a_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:26,217 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/7ef7e627cc2e4ebf9a89e15da0a6ae76, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:26,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/7ef7e627cc2e4ebf9a89e15da0a6ae76 is 175, key is test_row_0/A:col10/1694031263646/Put/seqid=0 2023-09-06T20:14:26,224 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/7ef7e627cc2e4ebf9a89e15da0a6ae76 2023-09-06T20:14:26,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/172d6e64ff624ad9a1898dd356739388 is 50, key is test_row_0/B:col10/1694031263646/Put/seqid=0 2023-09-06T20:14:26,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031326426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031326428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031326428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031326443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031326444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/172d6e64ff624ad9a1898dd356739388 2023-09-06T20:14:26,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/f195483370834c739b2c6b54f7ad75f6 is 50, key is test_row_0/C:col10/1694031263646/Put/seqid=0 2023-09-06T20:14:26,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031326932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031326932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031326939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031326945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:26,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031326950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:27,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/f195483370834c739b2c6b54f7ad75f6 2023-09-06T20:14:27,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/7ef7e627cc2e4ebf9a89e15da0a6ae76 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/7ef7e627cc2e4ebf9a89e15da0a6ae76 2023-09-06T20:14:27,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/7ef7e627cc2e4ebf9a89e15da0a6ae76, entries=200, sequenceid=331, filesize=39.0 K 2023-09-06T20:14:27,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/172d6e64ff624ad9a1898dd356739388 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/172d6e64ff624ad9a1898dd356739388 2023-09-06T20:14:27,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/172d6e64ff624ad9a1898dd356739388, entries=150, sequenceid=331, filesize=12.0 K 2023-09-06T20:14:27,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/f195483370834c739b2c6b54f7ad75f6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f195483370834c739b2c6b54f7ad75f6 2023-09-06T20:14:27,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f195483370834c739b2c6b54f7ad75f6, entries=150, sequenceid=331, filesize=12.0 K 2023-09-06T20:14:27,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=127.47 KB/130530 for 05a9474df993f762d37e597f1b39ed3e in 1295ms, sequenceid=331, compaction requested=true 2023-09-06T20:14:27,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:27,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:27,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:27,088 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:27,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:27,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:27,088 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:27,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:27,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:27,089 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103056 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:27,089 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/A is initiating minor compaction (all files) 2023-09-06T20:14:27,089 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/A in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:27,089 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/74044d2852bf4cbc860df1c97cc73974, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9adffabe33b44f01a661e2ce0f362079, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/7ef7e627cc2e4ebf9a89e15da0a6ae76] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=100.6 K 2023-09-06T20:14:27,089 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:27,089 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/74044d2852bf4cbc860df1c97cc73974, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9adffabe33b44f01a661e2ce0f362079, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/7ef7e627cc2e4ebf9a89e15da0a6ae76] 2023-09-06T20:14:27,090 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37500 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:27,090 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/B is initiating minor compaction (all files) 2023-09-06T20:14:27,090 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/B in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:27,090 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 74044d2852bf4cbc860df1c97cc73974, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1694031261886 2023-09-06T20:14:27,090 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/47055ab476684b72b15ea8f69c5638e4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/e0a4bed26dd6492bbb27674c711c896c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/172d6e64ff624ad9a1898dd356739388] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=36.6 K 2023-09-06T20:14:27,090 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9adffabe33b44f01a661e2ce0f362079, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1694031262506 2023-09-06T20:14:27,091 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 47055ab476684b72b15ea8f69c5638e4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1694031261886 2023-09-06T20:14:27,091 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7ef7e627cc2e4ebf9a89e15da0a6ae76, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1694031263636 2023-09-06T20:14:27,092 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting e0a4bed26dd6492bbb27674c711c896c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1694031262506 2023-09-06T20:14:27,092 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 172d6e64ff624ad9a1898dd356739388, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1694031263642 2023-09-06T20:14:27,102 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:27,103 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#B#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:27,103 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/31e938d652d94defb7c531a7ca6ad52c is 50, key is test_row_0/B:col10/1694031263646/Put/seqid=0 2023-09-06T20:14:27,118 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309064c0be2cc1c6745af9a9a39fd04896291_05a9474df993f762d37e597f1b39ed3e store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:27,119 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309064c0be2cc1c6745af9a9a39fd04896291_05a9474df993f762d37e597f1b39ed3e, store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:27,119 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309064c0be2cc1c6745af9a9a39fd04896291_05a9474df993f762d37e597f1b39ed3e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:27,152 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#A#compaction#200 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:27,153 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/6c44a5e83cb849d7a1decf4a9445b941 is 175, key is test_row_0/A:col10/1694031263646/Put/seqid=0 2023-09-06T20:14:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-06T20:14:27,164 INFO [Thread-855 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 58 completed 2023-09-06T20:14:27,165 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/31e938d652d94defb7c531a7ca6ad52c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/31e938d652d94defb7c531a7ca6ad52c 2023-09-06T20:14:27,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2023-09-06T20:14:27,169 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/6c44a5e83cb849d7a1decf4a9445b941 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/6c44a5e83cb849d7a1decf4a9445b941 2023-09-06T20:14:27,170 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-06T20:14:27,172 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:27,172 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:27,176 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/B of 05a9474df993f762d37e597f1b39ed3e into 31e938d652d94defb7c531a7ca6ad52c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:27,176 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:27,176 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/B, priority=13, startTime=1694031267088; duration=0sec 2023-09-06T20:14:27,176 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:27,176 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:B 2023-09-06T20:14:27,176 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:27,177 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37500 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:27,178 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/C is initiating minor compaction (all files) 2023-09-06T20:14:27,178 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/C in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:27,178 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/A of 05a9474df993f762d37e597f1b39ed3e into 6c44a5e83cb849d7a1decf4a9445b941(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:27,178 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:27,178 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/A, priority=13, startTime=1694031267088; duration=0sec 2023-09-06T20:14:27,178 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/e56dfd4493fb4801b778f21adbb339f0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/475bcf83842d4ff5a66059ae345500c7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f195483370834c739b2c6b54f7ad75f6] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=36.6 K 2023-09-06T20:14:27,178 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:27,178 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:A 2023-09-06T20:14:27,179 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting e56dfd4493fb4801b778f21adbb339f0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1694031261886 2023-09-06T20:14:27,179 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 475bcf83842d4ff5a66059ae345500c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1694031262506 2023-09-06T20:14:27,180 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting f195483370834c739b2c6b54f7ad75f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1694031263642 2023-09-06T20:14:27,186 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#C#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:27,187 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/fb62bb7714b6494cba39b3a885c3cc26 is 50, key is test_row_0/C:col10/1694031263646/Put/seqid=0 2023-09-06T20:14:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-06T20:14:27,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2023-09-06T20:14:27,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:27,327 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2023-09-06T20:14:27,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:27,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:27,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:27,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:27,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:27,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:27,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309061d3e6d7d06c14465a1c026f308b5f696_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031265813/Put/seqid=0 2023-09-06T20:14:27,343 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309061d3e6d7d06c14465a1c026f308b5f696_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309061d3e6d7d06c14465a1c026f308b5f696_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/758f75d81fa74a74bfec15df1de2e438, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:27,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/758f75d81fa74a74bfec15df1de2e438 is 175, key is test_row_0/A:col10/1694031265813/Put/seqid=0 2023-09-06T20:14:27,351 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/758f75d81fa74a74bfec15df1de2e438 2023-09-06T20:14:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/e0a64ba1384d454f829d14a28f103fdc is 50, key is test_row_0/B:col10/1694031265813/Put/seqid=0 2023-09-06T20:14:27,370 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/e0a64ba1384d454f829d14a28f103fdc 2023-09-06T20:14:27,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/86de9726f7b4406e94586ffa7eb8989f is 50, key is test_row_0/C:col10/1694031265813/Put/seqid=0 2023-09-06T20:14:27,386 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/86de9726f7b4406e94586ffa7eb8989f 2023-09-06T20:14:27,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/758f75d81fa74a74bfec15df1de2e438 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/758f75d81fa74a74bfec15df1de2e438 2023-09-06T20:14:27,397 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/758f75d81fa74a74bfec15df1de2e438, entries=150, sequenceid=355, filesize=30.5 K 2023-09-06T20:14:27,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/e0a64ba1384d454f829d14a28f103fdc as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/e0a64ba1384d454f829d14a28f103fdc 2023-09-06T20:14:27,406 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/e0a64ba1384d454f829d14a28f103fdc, entries=150, sequenceid=355, filesize=12.0 K 2023-09-06T20:14:27,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/86de9726f7b4406e94586ffa7eb8989f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/86de9726f7b4406e94586ffa7eb8989f 2023-09-06T20:14:27,423 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/86de9726f7b4406e94586ffa7eb8989f, entries=150, sequenceid=355, filesize=12.0 K 2023-09-06T20:14:27,425 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3022): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 05a9474df993f762d37e597f1b39ed3e in 97ms, sequenceid=355, compaction requested=false 2023-09-06T20:14:27,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:27,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:27,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=61 2023-09-06T20:14:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=61 2023-09-06T20:14:27,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=61, resume processing ppid=60 2023-09-06T20:14:27,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, ppid=60, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 254 msec 2023-09-06T20:14:27,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 262 msec 2023-09-06T20:14:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-06T20:14:27,475 INFO [Thread-855 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 60 completed 2023-09-06T20:14:27,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2023-09-06T20:14:27,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:14:27,482 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:27,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:14:27,620 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/fb62bb7714b6494cba39b3a885c3cc26 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/fb62bb7714b6494cba39b3a885c3cc26 2023-09-06T20:14:27,625 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/C of 05a9474df993f762d37e597f1b39ed3e into fb62bb7714b6494cba39b3a885c3cc26(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:27,625 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:27,625 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/C, priority=13, startTime=1694031267088; duration=0sec 2023-09-06T20:14:27,625 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:27,625 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:C 2023-09-06T20:14:27,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-06T20:14:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:27,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-09-06T20:14:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=63 2023-09-06T20:14:27,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-09-06T20:14:27,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 157 msec 2023-09-06T20:14:27,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 166 msec 2023-09-06T20:14:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:14:27,782 INFO [Thread-855 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 62 completed 2023-09-06T20:14:27,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2023-09-06T20:14:27,787 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:14:27,788 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:27,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:14:27,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2023-09-06T20:14:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-09-06T20:14:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=65 2023-09-06T20:14:27,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-09-06T20:14:27,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2023-09-06T20:14:27,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 161 msec 2023-09-06T20:14:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:27,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:14:27,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:27,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:27,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:27,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:27,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:27,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:27,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c70e56a600d74e4eac89d506aa64f45a_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031267983/Put/seqid=0 2023-09-06T20:14:28,004 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c70e56a600d74e4eac89d506aa64f45a_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c70e56a600d74e4eac89d506aa64f45a_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:28,005 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/b218cb8a27de45f6b1ae93f2c9357804, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:28,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/b218cb8a27de45f6b1ae93f2c9357804 is 175, key is test_row_0/A:col10/1694031267983/Put/seqid=0 2023-09-06T20:14:28,029 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/b218cb8a27de45f6b1ae93f2c9357804 2023-09-06T20:14:28,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/bf01cf4062734abca39eaa654262be42 is 50, key is test_row_0/B:col10/1694031267983/Put/seqid=0 2023-09-06T20:14:28,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/bf01cf4062734abca39eaa654262be42 2023-09-06T20:14:28,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/d0f2db66ed8c4e15a62cf63d86506160 is 50, key is test_row_0/C:col10/1694031267983/Put/seqid=0 2023-09-06T20:14:28,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031328064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031328065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031328066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031328066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031328067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:14:28,090 INFO [Thread-855 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 64 completed 2023-09-06T20:14:28,091 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2023-09-06T20:14:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:14:28,094 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:28,095 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:28,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:28,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031328168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031328168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031328169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031328169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031328169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:14:28,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2023-09-06T20:14:28,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:28,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:28,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:28,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(53): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:28,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:28,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031328371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031328371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031328371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031328372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031328372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:14:28,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2023-09-06T20:14:28,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:28,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:28,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:28,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(53): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:28,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:28,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/d0f2db66ed8c4e15a62cf63d86506160 2023-09-06T20:14:28,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/b218cb8a27de45f6b1ae93f2c9357804 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/b218cb8a27de45f6b1ae93f2c9357804 2023-09-06T20:14:28,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/b218cb8a27de45f6b1ae93f2c9357804, entries=400, sequenceid=369, filesize=73.0 K 2023-09-06T20:14:28,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/bf01cf4062734abca39eaa654262be42 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/bf01cf4062734abca39eaa654262be42 2023-09-06T20:14:28,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/bf01cf4062734abca39eaa654262be42, entries=400, sequenceid=369, filesize=23.9 K 2023-09-06T20:14:28,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/d0f2db66ed8c4e15a62cf63d86506160 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d0f2db66ed8c4e15a62cf63d86506160 2023-09-06T20:14:28,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d0f2db66ed8c4e15a62cf63d86506160, entries=100, sequenceid=369, filesize=9.6 K 2023-09-06T20:14:28,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 05a9474df993f762d37e597f1b39ed3e in 506ms, sequenceid=369, compaction requested=true 2023-09-06T20:14:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:28,492 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-06T20:14:28,492 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:28,494 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 138000 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:28,494 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/A is initiating minor compaction (all files) 2023-09-06T20:14:28,494 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/A in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:28,494 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/6c44a5e83cb849d7a1decf4a9445b941, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/758f75d81fa74a74bfec15df1de2e438, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/b218cb8a27de45f6b1ae93f2c9357804] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=134.8 K 2023-09-06T20:14:28,494 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:28,494 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/6c44a5e83cb849d7a1decf4a9445b941, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/758f75d81fa74a74bfec15df1de2e438, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/b218cb8a27de45f6b1ae93f2c9357804] 2023-09-06T20:14:28,495 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 49802 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:28,495 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/B is initiating minor compaction (all files) 2023-09-06T20:14:28,495 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/B in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:28,495 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/31e938d652d94defb7c531a7ca6ad52c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/e0a64ba1384d454f829d14a28f103fdc, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/bf01cf4062734abca39eaa654262be42] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=48.6 K 2023-09-06T20:14:28,495 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 6c44a5e83cb849d7a1decf4a9445b941, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1694031263642 2023-09-06T20:14:28,495 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 31e938d652d94defb7c531a7ca6ad52c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1694031263642 2023-09-06T20:14:28,497 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting e0a64ba1384d454f829d14a28f103fdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1694031265809 2023-09-06T20:14:28,497 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 758f75d81fa74a74bfec15df1de2e438, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1694031265809 2023-09-06T20:14:28,497 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting bf01cf4062734abca39eaa654262be42, keycount=400, bloomtype=ROW, size=23.9 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1694031267954 2023-09-06T20:14:28,498 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b218cb8a27de45f6b1ae93f2c9357804, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1694031267954 2023-09-06T20:14:28,506 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:28,515 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#B#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:28,516 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/c48e07eb88124ccaa7a0acfc7c1d8bae is 50, key is test_row_0/B:col10/1694031267983/Put/seqid=0 2023-09-06T20:14:28,518 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230906de27ee3bb4024448893a94ae9a5f8493_05a9474df993f762d37e597f1b39ed3e store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:28,521 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230906de27ee3bb4024448893a94ae9a5f8493_05a9474df993f762d37e597f1b39ed3e, store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:28,521 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906de27ee3bb4024448893a94ae9a5f8493_05a9474df993f762d37e597f1b39ed3e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:28,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2023-09-06T20:14:28,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:28,554 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-06T20:14:28,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:28,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:28,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:28,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:28,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:28,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:28,571 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#A#compaction#208 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:28,572 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/01869e5d1bee43bea84805152ac40dbd is 175, key is test_row_0/A:col10/1694031267983/Put/seqid=0 2023-09-06T20:14:28,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906bef080751a0c4f9a9e99cb87d5ae2470_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031268060/Put/seqid=0 2023-09-06T20:14:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:28,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. as already flushing 2023-09-06T20:14:28,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031328680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031328680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031328681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031328681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031328683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:14:28,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031328786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031328786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031328786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031328786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031328789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,953 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/c48e07eb88124ccaa7a0acfc7c1d8bae as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/c48e07eb88124ccaa7a0acfc7c1d8bae 2023-09-06T20:14:28,958 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/B of 05a9474df993f762d37e597f1b39ed3e into c48e07eb88124ccaa7a0acfc7c1d8bae(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:28,958 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:28,958 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/B, priority=13, startTime=1694031268492; duration=0sec 2023-09-06T20:14:28,959 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:28,959 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:B 2023-09-06T20:14:28,959 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:28,960 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35158 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:28,960 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 05a9474df993f762d37e597f1b39ed3e/C is initiating minor compaction (all files) 2023-09-06T20:14:28,960 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 05a9474df993f762d37e597f1b39ed3e/C in TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:28,960 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/fb62bb7714b6494cba39b3a885c3cc26, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/86de9726f7b4406e94586ffa7eb8989f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d0f2db66ed8c4e15a62cf63d86506160] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp, totalSize=34.3 K 2023-09-06T20:14:28,960 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting fb62bb7714b6494cba39b3a885c3cc26, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1694031263642 2023-09-06T20:14:28,961 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 86de9726f7b4406e94586ffa7eb8989f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1694031265809 2023-09-06T20:14:28,961 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting d0f2db66ed8c4e15a62cf63d86506160, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1694031267983 2023-09-06T20:14:28,976 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05a9474df993f762d37e597f1b39ed3e#C#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:28,976 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/4b7cf4142d5641ad81d080b0f9d62d22 is 50, key is test_row_0/C:col10/1694031267983/Put/seqid=0 2023-09-06T20:14:28,988 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/4b7cf4142d5641ad81d080b0f9d62d22 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/4b7cf4142d5641ad81d080b0f9d62d22 2023-09-06T20:14:28,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031328991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031328991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031328992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,994 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/C of 05a9474df993f762d37e597f1b39ed3e into 4b7cf4142d5641ad81d080b0f9d62d22(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:28,994 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:28,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,994 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/C, priority=13, startTime=1694031268492; duration=0sec 2023-09-06T20:14:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031328992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:28,994 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:28,995 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:C 2023-09-06T20:14:28,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031328992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:29,003 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/01869e5d1bee43bea84805152ac40dbd as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/01869e5d1bee43bea84805152ac40dbd 2023-09-06T20:14:29,006 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906bef080751a0c4f9a9e99cb87d5ae2470_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906bef080751a0c4f9a9e99cb87d5ae2470_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:29,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/9e37ac7934ba40b5a950148229603b01, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:29,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/9e37ac7934ba40b5a950148229603b01 is 175, key is test_row_0/A:col10/1694031268060/Put/seqid=0 2023-09-06T20:14:29,008 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 05a9474df993f762d37e597f1b39ed3e/A of 05a9474df993f762d37e597f1b39ed3e into 01869e5d1bee43bea84805152ac40dbd(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:29,008 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:29,008 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e., storeName=05a9474df993f762d37e597f1b39ed3e/A, priority=13, startTime=1694031268492; duration=0sec 2023-09-06T20:14:29,008 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:29,008 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:A 2023-09-06T20:14:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:14:29,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031329293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:29,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031329294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:29,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031329295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:29,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031329297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:29,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031329297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:29,417 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/9e37ac7934ba40b5a950148229603b01 2023-09-06T20:14:29,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/5329de4fe8a146da9e8aa787bc54a6f3 is 50, key is test_row_0/B:col10/1694031268060/Put/seqid=0 2023-09-06T20:14:29,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37438 deadline: 1694031329797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:29,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37382 deadline: 1694031329797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:29,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37430 deadline: 1694031329798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:29,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:55174 deadline: 1694031329800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:29,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37448 deadline: 1694031329800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:29,832 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/5329de4fe8a146da9e8aa787bc54a6f3 2023-09-06T20:14:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/d40f3250d34f40d98a6ab50b6221a1fc is 50, key is test_row_0/C:col10/1694031268060/Put/seqid=0 2023-09-06T20:14:29,854 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/d40f3250d34f40d98a6ab50b6221a1fc 2023-09-06T20:14:29,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/9e37ac7934ba40b5a950148229603b01 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9e37ac7934ba40b5a950148229603b01 2023-09-06T20:14:29,866 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9e37ac7934ba40b5a950148229603b01, entries=150, sequenceid=395, filesize=30.5 K 2023-09-06T20:14:29,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/5329de4fe8a146da9e8aa787bc54a6f3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/5329de4fe8a146da9e8aa787bc54a6f3 2023-09-06T20:14:29,875 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/5329de4fe8a146da9e8aa787bc54a6f3, entries=150, sequenceid=395, filesize=12.0 K 2023-09-06T20:14:29,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/d40f3250d34f40d98a6ab50b6221a1fc as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d40f3250d34f40d98a6ab50b6221a1fc 2023-09-06T20:14:29,883 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d40f3250d34f40d98a6ab50b6221a1fc, entries=150, sequenceid=395, filesize=12.0 K 2023-09-06T20:14:29,884 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 05a9474df993f762d37e597f1b39ed3e in 1329ms, sequenceid=395, compaction requested=false 2023-09-06T20:14:29,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:29,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:29,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=67 2023-09-06T20:14:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=67 2023-09-06T20:14:29,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-09-06T20:14:29,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7900 sec 2023-09-06T20:14:29,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.7980 sec 2023-09-06T20:14:30,158 DEBUG [Thread-858 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x64362822 to 127.0.0.1:58481 2023-09-06T20:14:30,158 DEBUG [Thread-858 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:30,159 DEBUG [Thread-856 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5cc3635d to 127.0.0.1:58481 2023-09-06T20:14:30,159 DEBUG [Thread-856 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:30,167 DEBUG [Thread-860 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2cda7fd4 to 127.0.0.1:58481 2023-09-06T20:14:30,167 DEBUG [Thread-860 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:30,171 DEBUG [Thread-862 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6a78a12d to 127.0.0.1:58481 2023-09-06T20:14:30,171 DEBUG [Thread-862 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:14:30,199 INFO [Thread-855 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 66 completed 2023-09-06T20:14:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:30,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:14:30,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:30,802 DEBUG [Thread-853 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x54a7d44c to 127.0.0.1:58481 2023-09-06T20:14:30,802 DEBUG [Thread-853 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:30,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:30,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:30,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:30,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:30,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:30,806 DEBUG [Thread-849 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6da744d5 to 127.0.0.1:58481 2023-09-06T20:14:30,806 DEBUG [Thread-849 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:30,807 DEBUG [Thread-845 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x773a3d1e to 127.0.0.1:58481 2023-09-06T20:14:30,808 DEBUG [Thread-845 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:30,810 DEBUG [Thread-847 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x37152f12 to 127.0.0.1:58481 2023-09-06T20:14:30,810 DEBUG [Thread-847 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:30,812 DEBUG [Thread-851 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1e0a1074 to 127.0.0.1:58481 2023-09-06T20:14:30,812 DEBUG [Thread-851 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:30,812 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2023-09-06T20:14:30,812 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2023-09-06T20:14:30,812 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2023-09-06T20:14:30,812 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 131 2023-09-06T20:14:30,812 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2023-09-06T20:14:30,812 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2023-09-06T20:14:30,812 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2023-09-06T20:14:30,813 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(398): read 5957 2023-09-06T20:14:30,813 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(398): read 5899 2023-09-06T20:14:30,813 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2023-09-06T20:14:30,813 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(402): scanned 2665 2023-09-06T20:14:30,813 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(403): verified 7995 rows 2023-09-06T20:14:30,813 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(402): scanned 2687 2023-09-06T20:14:30,813 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(403): verified 8058 rows 2023-09-06T20:14:30,813 INFO [Listener at localhost/35961 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:14:30,813 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6c3ba4b6 to 127.0.0.1:58481 2023-09-06T20:14:30,813 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:30,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906ad56fd38a2cf4602a6a4239eee26a527_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_0/A:col10/1694031268682/Put/seqid=0 2023-09-06T20:14:30,835 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2023-09-06T20:14:30,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable TestAcidGuarantees 2023-09-06T20:14:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=68, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:14:30,844 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031270844"}]},"ts":"1694031270844"} 2023-09-06T20:14:30,846 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2023-09-06T20:14:30,923 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2023-09-06T20:14:30,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=05a9474df993f762d37e597f1b39ed3e, UNASSIGN}] 2023-09-06T20:14:30,925 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=05a9474df993f762d37e597f1b39ed3e, UNASSIGN 2023-09-06T20:14:30,925 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=05a9474df993f762d37e597f1b39ed3e, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:30,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE; CloseRegionProcedure 05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:14:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:14:31,078 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(119): Close 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:31,078 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1668): Closing 05a9474df993f762d37e597f1b39ed3e, disabling compactions & flushes 2023-09-06T20:14:31,078 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1929): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:14:31,266 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906ad56fd38a2cf4602a6a4239eee26a527_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906ad56fd38a2cf4602a6a4239eee26a527_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:31,267 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/1fc5a195cd014f40b868aa2b9f0c3c59, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:31,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/1fc5a195cd014f40b868aa2b9f0c3c59 is 175, key is test_row_0/A:col10/1694031268682/Put/seqid=0 2023-09-06T20:14:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:14:31,672 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/1fc5a195cd014f40b868aa2b9f0c3c59 2023-09-06T20:14:31,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/eea3041f007e416ba303172700eff7fe is 50, key is test_row_0/B:col10/1694031268682/Put/seqid=0 2023-09-06T20:14:31,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:14:32,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/eea3041f007e416ba303172700eff7fe 2023-09-06T20:14:32,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/eb7b76c373ee4e2abdbc0d5b1de0476c is 50, key is test_row_0/C:col10/1694031268682/Put/seqid=0 2023-09-06T20:14:32,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/eb7b76c373ee4e2abdbc0d5b1de0476c 2023-09-06T20:14:32,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/1fc5a195cd014f40b868aa2b9f0c3c59 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/1fc5a195cd014f40b868aa2b9f0c3c59 2023-09-06T20:14:32,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/1fc5a195cd014f40b868aa2b9f0c3c59, entries=150, sequenceid=409, filesize=30.5 K 2023-09-06T20:14:32,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/eea3041f007e416ba303172700eff7fe as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/eea3041f007e416ba303172700eff7fe 2023-09-06T20:14:32,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/eea3041f007e416ba303172700eff7fe, entries=150, sequenceid=409, filesize=12.0 K 2023-09-06T20:14:32,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/eb7b76c373ee4e2abdbc0d5b1de0476c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/eb7b76c373ee4e2abdbc0d5b1de0476c 2023-09-06T20:14:32,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/eb7b76c373ee4e2abdbc0d5b1de0476c, entries=150, sequenceid=409, filesize=12.0 K 2023-09-06T20:14:32,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=26.84 KB/27480 for 05a9474df993f762d37e597f1b39ed3e in 1709ms, sequenceid=409, compaction requested=true 2023-09-06T20:14:32,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:32,512 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:32,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:32,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:32,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. after waiting 0 ms 2023-09-06T20:14:32,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:32,512 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. because compaction request was cancelled 2023-09-06T20:14:32,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:32,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:32,512 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:A 2023-09-06T20:14:32,512 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. because compaction request was cancelled 2023-09-06T20:14:32,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:32,512 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(2819): Flushing 05a9474df993f762d37e597f1b39ed3e 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2023-09-06T20:14:32,512 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:B 2023-09-06T20:14:32,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05a9474df993f762d37e597f1b39ed3e:C, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:32,512 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. because compaction request was cancelled 2023-09-06T20:14:32,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:32,512 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05a9474df993f762d37e597f1b39ed3e:C 2023-09-06T20:14:32,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=A 2023-09-06T20:14:32,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:32,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=B 2023-09-06T20:14:32,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:32,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 05a9474df993f762d37e597f1b39ed3e, store=C 2023-09-06T20:14:32,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:32,517 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090668876401573e4eafb041491e9cdd0a20_05a9474df993f762d37e597f1b39ed3e is 50, key is test_row_1/A:col10/1694031270810/Put/seqid=0 2023-09-06T20:14:32,524 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090668876401573e4eafb041491e9cdd0a20_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090668876401573e4eafb041491e9cdd0a20_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:32,525 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/e9aabc0857f54d80ae5a258e28e1e831, store: [table=TestAcidGuarantees family=A region=05a9474df993f762d37e597f1b39ed3e] 2023-09-06T20:14:32,526 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/e9aabc0857f54d80ae5a258e28e1e831 is 175, key is test_row_1/A:col10/1694031270810/Put/seqid=0 2023-09-06T20:14:32,536 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=416, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/e9aabc0857f54d80ae5a258e28e1e831 2023-09-06T20:14:32,542 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/f966bfdd80ec4afca538ad3d9cf62948 is 50, key is test_row_1/B:col10/1694031270810/Put/seqid=0 2023-09-06T20:14:32,947 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/f966bfdd80ec4afca538ad3d9cf62948 2023-09-06T20:14:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:14:32,953 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/e8c3f39186ff4ddfb28b5f76707997f4 is 50, key is test_row_1/C:col10/1694031270810/Put/seqid=0 2023-09-06T20:14:33,357 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/e8c3f39186ff4ddfb28b5f76707997f4 2023-09-06T20:14:33,361 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/A/e9aabc0857f54d80ae5a258e28e1e831 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/e9aabc0857f54d80ae5a258e28e1e831 2023-09-06T20:14:33,364 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/e9aabc0857f54d80ae5a258e28e1e831, entries=50, sequenceid=416, filesize=13.5 K 2023-09-06T20:14:33,365 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/B/f966bfdd80ec4afca538ad3d9cf62948 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/f966bfdd80ec4afca538ad3d9cf62948 2023-09-06T20:14:33,369 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/f966bfdd80ec4afca538ad3d9cf62948, entries=50, sequenceid=416, filesize=7.2 K 2023-09-06T20:14:33,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/.tmp/C/e8c3f39186ff4ddfb28b5f76707997f4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/e8c3f39186ff4ddfb28b5f76707997f4 2023-09-06T20:14:33,373 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/e8c3f39186ff4ddfb28b5f76707997f4, entries=50, sequenceid=416, filesize=7.2 K 2023-09-06T20:14:33,374 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(3022): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 05a9474df993f762d37e597f1b39ed3e in 862ms, sequenceid=416, compaction requested=true 2023-09-06T20:14:33,395 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/f22c95452e1741e68e14372644832446, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/edcbda4b44e842f6ad7b5477946fac8f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/a5904564161f4e76adc1584e055fedf5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/c4e0486c1e4a4558aa867d4a9d028da0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/080e7540ba7244f294fa4d31aa1d589c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/7d2533ddf5644d7f89e39f593fb7491f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/d7a70d40c93048c5b52f5a5093fe4794, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/08cd7f180448459f9082203e9053e04a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/6454e248f41e4aba8bc770c10da3af09, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/884d93b87f9c408fb277698e1b75337a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/2c5dc3431a694d5cac62406054923e5f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/a2777eaa81ce4567b55ec48a88b33092, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/df8ea28dd894438588115a8f8a2a14ab, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/d525e3ec7b7a415f934596b18dfea79e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/1e14fdf24cfe42faa92c0cb0d2b879c4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/b2a42e3cd8e442418b46bb824c021225, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/cf188553c6f045ca91cbd25389a84c7c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/4541243c942a4524870d06729d4b5b02, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/74044d2852bf4cbc860df1c97cc73974, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9d92854eb936450f9233bd36da58b42d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9adffabe33b44f01a661e2ce0f362079, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/7ef7e627cc2e4ebf9a89e15da0a6ae76, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/6c44a5e83cb849d7a1decf4a9445b941, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/758f75d81fa74a74bfec15df1de2e438, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/b218cb8a27de45f6b1ae93f2c9357804] to archive 2023-09-06T20:14:33,396 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:14:33,398 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/f22c95452e1741e68e14372644832446 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/f22c95452e1741e68e14372644832446 2023-09-06T20:14:33,399 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/edcbda4b44e842f6ad7b5477946fac8f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/edcbda4b44e842f6ad7b5477946fac8f 2023-09-06T20:14:33,400 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/a5904564161f4e76adc1584e055fedf5 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/a5904564161f4e76adc1584e055fedf5 2023-09-06T20:14:33,402 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/c4e0486c1e4a4558aa867d4a9d028da0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/c4e0486c1e4a4558aa867d4a9d028da0 2023-09-06T20:14:33,403 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/080e7540ba7244f294fa4d31aa1d589c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/080e7540ba7244f294fa4d31aa1d589c 2023-09-06T20:14:33,404 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/7d2533ddf5644d7f89e39f593fb7491f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/7d2533ddf5644d7f89e39f593fb7491f 2023-09-06T20:14:33,405 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/d7a70d40c93048c5b52f5a5093fe4794 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/d7a70d40c93048c5b52f5a5093fe4794 2023-09-06T20:14:33,406 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/08cd7f180448459f9082203e9053e04a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/08cd7f180448459f9082203e9053e04a 2023-09-06T20:14:33,407 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/6454e248f41e4aba8bc770c10da3af09 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/6454e248f41e4aba8bc770c10da3af09 2023-09-06T20:14:33,408 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/884d93b87f9c408fb277698e1b75337a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/884d93b87f9c408fb277698e1b75337a 2023-09-06T20:14:33,409 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/2c5dc3431a694d5cac62406054923e5f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/2c5dc3431a694d5cac62406054923e5f 2023-09-06T20:14:33,410 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/a2777eaa81ce4567b55ec48a88b33092 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/a2777eaa81ce4567b55ec48a88b33092 2023-09-06T20:14:33,411 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/df8ea28dd894438588115a8f8a2a14ab to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/df8ea28dd894438588115a8f8a2a14ab 2023-09-06T20:14:33,412 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/d525e3ec7b7a415f934596b18dfea79e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/d525e3ec7b7a415f934596b18dfea79e 2023-09-06T20:14:33,413 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/1e14fdf24cfe42faa92c0cb0d2b879c4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/1e14fdf24cfe42faa92c0cb0d2b879c4 2023-09-06T20:14:33,414 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/b2a42e3cd8e442418b46bb824c021225 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/b2a42e3cd8e442418b46bb824c021225 2023-09-06T20:14:33,415 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/cf188553c6f045ca91cbd25389a84c7c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/cf188553c6f045ca91cbd25389a84c7c 2023-09-06T20:14:33,416 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/4541243c942a4524870d06729d4b5b02 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/4541243c942a4524870d06729d4b5b02 2023-09-06T20:14:33,417 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/74044d2852bf4cbc860df1c97cc73974 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/74044d2852bf4cbc860df1c97cc73974 2023-09-06T20:14:33,418 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9d92854eb936450f9233bd36da58b42d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9d92854eb936450f9233bd36da58b42d 2023-09-06T20:14:33,419 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9adffabe33b44f01a661e2ce0f362079 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9adffabe33b44f01a661e2ce0f362079 2023-09-06T20:14:33,421 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/7ef7e627cc2e4ebf9a89e15da0a6ae76 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/7ef7e627cc2e4ebf9a89e15da0a6ae76 2023-09-06T20:14:33,422 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/6c44a5e83cb849d7a1decf4a9445b941 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/6c44a5e83cb849d7a1decf4a9445b941 2023-09-06T20:14:33,423 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/758f75d81fa74a74bfec15df1de2e438 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/758f75d81fa74a74bfec15df1de2e438 2023-09-06T20:14:33,424 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/b218cb8a27de45f6b1ae93f2c9357804 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/b218cb8a27de45f6b1ae93f2c9357804 2023-09-06T20:14:33,426 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/106b73fbee5e450e8ccec05309f97344, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/a0547b74e1f14a2898ae944ff59d5a9f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/45761bc470b04adbb81e05f5782ffc5e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/94a2493a0ce344ea9118433236712888, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/85d6a3d1f31a44698262f045d0af11ca, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/ed06e68dea4046ff9b88cc4ec8e4f6a0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/6b74138bb5714ad79d97c33c56ee70b7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/16916d2e938f428a8cf1bd947094b271, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/5481b594f5d048ea82cb8b84f8ead42a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/0ba3a42582ee4f9d808caa20a7d9e92c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/eb7162e990704e1b89d1147d4e85dc9c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/55f6979fe92b41608f62709ce6425d2b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/ec697cff18d0460da8705cada7280f9d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/f966a0a90b724f7a8a317c5a117553f8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/8ca5ccb2db2445259c1b9060a9266558, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/68a4962d4ae3468ea01869111f6e5ebe, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/9e1bac5bd71a47a2a3fa2edccaa08746, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/787f4687c6f24b419b947fc397db782b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/47055ab476684b72b15ea8f69c5638e4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/829465f90e2a4cffad250109c42db172, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/e0a4bed26dd6492bbb27674c711c896c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/31e938d652d94defb7c531a7ca6ad52c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/172d6e64ff624ad9a1898dd356739388, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/e0a64ba1384d454f829d14a28f103fdc, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/bf01cf4062734abca39eaa654262be42] to archive 2023-09-06T20:14:33,427 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:14:33,428 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/106b73fbee5e450e8ccec05309f97344 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/106b73fbee5e450e8ccec05309f97344 2023-09-06T20:14:33,430 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/a0547b74e1f14a2898ae944ff59d5a9f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/a0547b74e1f14a2898ae944ff59d5a9f 2023-09-06T20:14:33,431 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/45761bc470b04adbb81e05f5782ffc5e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/45761bc470b04adbb81e05f5782ffc5e 2023-09-06T20:14:33,432 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/94a2493a0ce344ea9118433236712888 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/94a2493a0ce344ea9118433236712888 2023-09-06T20:14:33,433 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/85d6a3d1f31a44698262f045d0af11ca to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/85d6a3d1f31a44698262f045d0af11ca 2023-09-06T20:14:33,434 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/ed06e68dea4046ff9b88cc4ec8e4f6a0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/ed06e68dea4046ff9b88cc4ec8e4f6a0 2023-09-06T20:14:33,435 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/6b74138bb5714ad79d97c33c56ee70b7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/6b74138bb5714ad79d97c33c56ee70b7 2023-09-06T20:14:33,436 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/16916d2e938f428a8cf1bd947094b271 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/16916d2e938f428a8cf1bd947094b271 2023-09-06T20:14:33,437 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/5481b594f5d048ea82cb8b84f8ead42a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/5481b594f5d048ea82cb8b84f8ead42a 2023-09-06T20:14:33,438 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/0ba3a42582ee4f9d808caa20a7d9e92c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/0ba3a42582ee4f9d808caa20a7d9e92c 2023-09-06T20:14:33,439 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/eb7162e990704e1b89d1147d4e85dc9c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/eb7162e990704e1b89d1147d4e85dc9c 2023-09-06T20:14:33,440 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/55f6979fe92b41608f62709ce6425d2b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/55f6979fe92b41608f62709ce6425d2b 2023-09-06T20:14:33,441 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/ec697cff18d0460da8705cada7280f9d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/ec697cff18d0460da8705cada7280f9d 2023-09-06T20:14:33,442 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/f966a0a90b724f7a8a317c5a117553f8 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/f966a0a90b724f7a8a317c5a117553f8 2023-09-06T20:14:33,443 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/8ca5ccb2db2445259c1b9060a9266558 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/8ca5ccb2db2445259c1b9060a9266558 2023-09-06T20:14:33,444 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/68a4962d4ae3468ea01869111f6e5ebe to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/68a4962d4ae3468ea01869111f6e5ebe 2023-09-06T20:14:33,445 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/9e1bac5bd71a47a2a3fa2edccaa08746 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/9e1bac5bd71a47a2a3fa2edccaa08746 2023-09-06T20:14:33,446 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/787f4687c6f24b419b947fc397db782b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/787f4687c6f24b419b947fc397db782b 2023-09-06T20:14:33,447 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/47055ab476684b72b15ea8f69c5638e4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/47055ab476684b72b15ea8f69c5638e4 2023-09-06T20:14:33,448 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/829465f90e2a4cffad250109c42db172 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/829465f90e2a4cffad250109c42db172 2023-09-06T20:14:33,449 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/e0a4bed26dd6492bbb27674c711c896c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/e0a4bed26dd6492bbb27674c711c896c 2023-09-06T20:14:33,450 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/31e938d652d94defb7c531a7ca6ad52c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/31e938d652d94defb7c531a7ca6ad52c 2023-09-06T20:14:33,451 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/172d6e64ff624ad9a1898dd356739388 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/172d6e64ff624ad9a1898dd356739388 2023-09-06T20:14:33,452 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/e0a64ba1384d454f829d14a28f103fdc to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/e0a64ba1384d454f829d14a28f103fdc 2023-09-06T20:14:33,453 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/bf01cf4062734abca39eaa654262be42 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/bf01cf4062734abca39eaa654262be42 2023-09-06T20:14:33,456 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/78d8b51d6de94b72a2cd506b915aecc4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/fe74dce9198a4f46a16bcf999d6176a6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d4944075f43e4790a5effd4fd70d0033, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f4c750df0b7944f5b599f834a6565654, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/bde37ef3020c42f58cac2a5012301c08, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f61968c64db14e2c97fb2663567a2799, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/83fb82e52c92410aaef2f99867b6c64c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/01643590d0af462fb050a11527852039, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/672a776a69e84959b0faf914b1259a23, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/19983c582907468fada196b7ff5120c0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/53c1b79c70fa4660898e11e63da51d0f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/33ec0ebf76144b129b331be1ebf2185b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/5e6d3b64ffbb4f3cbe57823619420dad, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f4272f452c5b4c9581e30fedf50ecc7c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/54282fe9b0bc46c1bbe49dfefc604951, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/947142655eba46d28f346cff822b26fd, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/59e0b5988e204f7988661f13e5a70314, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/1a8ccffcae234f2f880ef87cbe3626e5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/e56dfd4493fb4801b778f21adbb339f0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/a1f08d9a777b4d0fbd0359c7e9cf2963, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/475bcf83842d4ff5a66059ae345500c7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/fb62bb7714b6494cba39b3a885c3cc26, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f195483370834c739b2c6b54f7ad75f6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/86de9726f7b4406e94586ffa7eb8989f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d0f2db66ed8c4e15a62cf63d86506160] to archive 2023-09-06T20:14:33,456 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:14:33,458 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/78d8b51d6de94b72a2cd506b915aecc4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/78d8b51d6de94b72a2cd506b915aecc4 2023-09-06T20:14:33,459 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/fe74dce9198a4f46a16bcf999d6176a6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/fe74dce9198a4f46a16bcf999d6176a6 2023-09-06T20:14:33,460 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d4944075f43e4790a5effd4fd70d0033 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d4944075f43e4790a5effd4fd70d0033 2023-09-06T20:14:33,461 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f4c750df0b7944f5b599f834a6565654 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f4c750df0b7944f5b599f834a6565654 2023-09-06T20:14:33,462 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/bde37ef3020c42f58cac2a5012301c08 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/bde37ef3020c42f58cac2a5012301c08 2023-09-06T20:14:33,463 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f61968c64db14e2c97fb2663567a2799 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f61968c64db14e2c97fb2663567a2799 2023-09-06T20:14:33,464 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/83fb82e52c92410aaef2f99867b6c64c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/83fb82e52c92410aaef2f99867b6c64c 2023-09-06T20:14:33,465 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/01643590d0af462fb050a11527852039 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/01643590d0af462fb050a11527852039 2023-09-06T20:14:33,467 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/672a776a69e84959b0faf914b1259a23 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/672a776a69e84959b0faf914b1259a23 2023-09-06T20:14:33,468 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/19983c582907468fada196b7ff5120c0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/19983c582907468fada196b7ff5120c0 2023-09-06T20:14:33,469 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/53c1b79c70fa4660898e11e63da51d0f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/53c1b79c70fa4660898e11e63da51d0f 2023-09-06T20:14:33,470 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/33ec0ebf76144b129b331be1ebf2185b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/33ec0ebf76144b129b331be1ebf2185b 2023-09-06T20:14:33,471 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/5e6d3b64ffbb4f3cbe57823619420dad to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/5e6d3b64ffbb4f3cbe57823619420dad 2023-09-06T20:14:33,472 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f4272f452c5b4c9581e30fedf50ecc7c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f4272f452c5b4c9581e30fedf50ecc7c 2023-09-06T20:14:33,473 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/54282fe9b0bc46c1bbe49dfefc604951 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/54282fe9b0bc46c1bbe49dfefc604951 2023-09-06T20:14:33,474 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/947142655eba46d28f346cff822b26fd to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/947142655eba46d28f346cff822b26fd 2023-09-06T20:14:33,475 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/59e0b5988e204f7988661f13e5a70314 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/59e0b5988e204f7988661f13e5a70314 2023-09-06T20:14:33,476 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/1a8ccffcae234f2f880ef87cbe3626e5 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/1a8ccffcae234f2f880ef87cbe3626e5 2023-09-06T20:14:33,477 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/e56dfd4493fb4801b778f21adbb339f0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/e56dfd4493fb4801b778f21adbb339f0 2023-09-06T20:14:33,478 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/a1f08d9a777b4d0fbd0359c7e9cf2963 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/a1f08d9a777b4d0fbd0359c7e9cf2963 2023-09-06T20:14:33,480 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/475bcf83842d4ff5a66059ae345500c7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/475bcf83842d4ff5a66059ae345500c7 2023-09-06T20:14:33,481 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/fb62bb7714b6494cba39b3a885c3cc26 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/fb62bb7714b6494cba39b3a885c3cc26 2023-09-06T20:14:33,482 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f195483370834c739b2c6b54f7ad75f6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/f195483370834c739b2c6b54f7ad75f6 2023-09-06T20:14:33,483 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/86de9726f7b4406e94586ffa7eb8989f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/86de9726f7b4406e94586ffa7eb8989f 2023-09-06T20:14:33,484 DEBUG [StoreCloser-TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d0f2db66ed8c4e15a62cf63d86506160 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d0f2db66ed8c4e15a62cf63d86506160 2023-09-06T20:14:33,494 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/recovered.edits/419.seqid, newMaxSeqId=419, maxSeqId=4 2023-09-06T20:14:33,496 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e. 2023-09-06T20:14:33,496 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1622): Region close journal for 05a9474df993f762d37e597f1b39ed3e: 2023-09-06T20:14:33,498 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(163): Closed 05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:33,499 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=05a9474df993f762d37e597f1b39ed3e, regionState=CLOSED 2023-09-06T20:14:33,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=70, resume processing ppid=69 2023-09-06T20:14:33,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, ppid=69, state=SUCCESS; CloseRegionProcedure 05a9474df993f762d37e597f1b39ed3e, server=jenkins-hbase3.apache.org,33275,1694031217268 in 2.5740 sec 2023-09-06T20:14:33,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=69, resume processing ppid=68 2023-09-06T20:14:33,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, ppid=68, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=05a9474df993f762d37e597f1b39ed3e, UNASSIGN in 2.5780 sec 2023-09-06T20:14:33,504 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031273503"}]},"ts":"1694031273503"} 2023-09-06T20:14:33,504 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2023-09-06T20:14:33,506 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2023-09-06T20:14:33,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 2.6710 sec 2023-09-06T20:14:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:14:34,949 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 68 completed 2023-09-06T20:14:34,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete TestAcidGuarantees 2023-09-06T20:14:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=71, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:34,952 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=71, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-09-06T20:14:34,952 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=71, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:34,955 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:34,959 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/recovered.edits] 2023-09-06T20:14:34,962 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/01869e5d1bee43bea84805152ac40dbd to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/01869e5d1bee43bea84805152ac40dbd 2023-09-06T20:14:34,964 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/1fc5a195cd014f40b868aa2b9f0c3c59 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/1fc5a195cd014f40b868aa2b9f0c3c59 2023-09-06T20:14:34,966 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9e37ac7934ba40b5a950148229603b01 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/9e37ac7934ba40b5a950148229603b01 2023-09-06T20:14:34,967 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/e9aabc0857f54d80ae5a258e28e1e831 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/A/e9aabc0857f54d80ae5a258e28e1e831 2023-09-06T20:14:34,970 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/5329de4fe8a146da9e8aa787bc54a6f3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/5329de4fe8a146da9e8aa787bc54a6f3 2023-09-06T20:14:34,971 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/c48e07eb88124ccaa7a0acfc7c1d8bae to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/c48e07eb88124ccaa7a0acfc7c1d8bae 2023-09-06T20:14:34,972 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/eea3041f007e416ba303172700eff7fe to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/eea3041f007e416ba303172700eff7fe 2023-09-06T20:14:34,973 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/f966bfdd80ec4afca538ad3d9cf62948 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/B/f966bfdd80ec4afca538ad3d9cf62948 2023-09-06T20:14:34,975 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/4b7cf4142d5641ad81d080b0f9d62d22 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/4b7cf4142d5641ad81d080b0f9d62d22 2023-09-06T20:14:34,977 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d40f3250d34f40d98a6ab50b6221a1fc to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/d40f3250d34f40d98a6ab50b6221a1fc 2023-09-06T20:14:34,978 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/e8c3f39186ff4ddfb28b5f76707997f4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/e8c3f39186ff4ddfb28b5f76707997f4 2023-09-06T20:14:34,979 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/eb7b76c373ee4e2abdbc0d5b1de0476c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/C/eb7b76c373ee4e2abdbc0d5b1de0476c 2023-09-06T20:14:34,982 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/recovered.edits/419.seqid to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e/recovered.edits/419.seqid 2023-09-06T20:14:34,983 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:34,983 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2023-09-06T20:14:34,983 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2023-09-06T20:14:34,984 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2023-09-06T20:14:34,988 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090615d98b8006064f8c87e10cef920d4eaa_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090615d98b8006064f8c87e10cef920d4eaa_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:34,989 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309061d3e6d7d06c14465a1c026f308b5f696_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309061d3e6d7d06c14465a1c026f308b5f696_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:34,990 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090630c2370529264841ab2fec5a63b66423_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090630c2370529264841ab2fec5a63b66423_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:34,991 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090643362edc89ef498fb6aede7a10488383_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090643362edc89ef498fb6aede7a10488383_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:34,992 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090643f0efca52714d29bab993d3eb512485_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090643f0efca52714d29bab993d3eb512485_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:34,993 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309064c86ced9476342a8a3cf70bdcd777f3b_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309064c86ced9476342a8a3cf70bdcd777f3b_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:34,995 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309065cec533a7cb440e6b3278e9887eb642a_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309065cec533a7cb440e6b3278e9887eb642a_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:34,996 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309065ff2cc70e0724e0690705d4dbe6a6476_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309065ff2cc70e0724e0690705d4dbe6a6476_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:34,997 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090668876401573e4eafb041491e9cdd0a20_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090668876401573e4eafb041491e9cdd0a20_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:34,998 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309066a6a03e0fe1a489c98e2c3f6920184b1_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309066a6a03e0fe1a489c98e2c3f6920184b1_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:34,999 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309068c243d07d59a4781b6a890688b322274_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309068c243d07d59a4781b6a890688b322274_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:35,000 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309069c38c0fc51614352af481f32257d5e4a_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309069c38c0fc51614352af481f32257d5e4a_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:35,001 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906ad56fd38a2cf4602a6a4239eee26a527_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906ad56fd38a2cf4602a6a4239eee26a527_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:35,003 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906bef080751a0c4f9a9e99cb87d5ae2470_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906bef080751a0c4f9a9e99cb87d5ae2470_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:35,004 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c5ffc7a8d9d74b07b84b188738ad68c9_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c5ffc7a8d9d74b07b84b188738ad68c9_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:35,005 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c64592adebdb4dae8fb07e344ceb43d1_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c64592adebdb4dae8fb07e344ceb43d1_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:35,006 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c70e56a600d74e4eac89d506aa64f45a_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c70e56a600d74e4eac89d506aa64f45a_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:35,007 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906cdf11344ad2149b8b380d32dd119b26e_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906cdf11344ad2149b8b380d32dd119b26e_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:35,008 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906cfea31ac0d7f4612b2c34ebbcbf8ba33_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906cfea31ac0d7f4612b2c34ebbcbf8ba33_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:35,009 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906e930b740e0fa4460a5d99e37e6da0c13_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906e930b740e0fa4460a5d99e37e6da0c13_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:35,010 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906ee8ce9c5e03442c682383051459e1267_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906ee8ce9c5e03442c682383051459e1267_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:35,011 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906fd9e6b6cdd6b452f935359cef684706f_05a9474df993f762d37e597f1b39ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906fd9e6b6cdd6b452f935359cef684706f_05a9474df993f762d37e597f1b39ed3e 2023-09-06T20:14:35,012 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2023-09-06T20:14:35,014 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=71, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:35,017 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2023-09-06T20:14:35,019 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2023-09-06T20:14:35,020 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=71, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:35,020 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2023-09-06T20:14:35,020 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694031275020"}]},"ts":"9223372036854775807"} 2023-09-06T20:14:35,021 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:14:35,021 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 05a9474df993f762d37e597f1b39ed3e, NAME => 'TestAcidGuarantees,,1694031246995.05a9474df993f762d37e597f1b39ed3e.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:14:35,021 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2023-09-06T20:14:35,021 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694031275021"}]},"ts":"9223372036854775807"} 2023-09-06T20:14:35,023 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2023-09-06T20:14:35,025 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(127): Finished pid=71, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:35,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 75 msec 2023-09-06T20:14:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-09-06T20:14:35,053 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 71 completed 2023-09-06T20:14:35,068 INFO [Listener at localhost/35961 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testMobMixedAtomicity Thread=221 (was 223), OpenFileDescriptor=576 (was 652), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=559 (was 518) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 170), AvailableMemoryMB=5037 (was 5138) 2023-09-06T20:14:35,082 INFO [Listener at localhost/35961 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testGetAtomicity Thread=221, OpenFileDescriptor=576, MaxFileDescriptor=60000, SystemLoadAverage=559, ProcessCount=170, AvailableMemoryMB=5036 2023-09-06T20:14:35,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-06T20:14:35,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:14:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2023-09-06T20:14:35,086 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:14:35,086 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:14:35,087 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:14:35,087 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 72 2023-09-06T20:14:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:14:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:14:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:14:35,497 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236 2023-09-06T20:14:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:14:35,903 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:14:35,903 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1668): Closing e66eccf326cb10336c74bdb6873cf7b9, disabling compactions & flushes 2023-09-06T20:14:35,903 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:35,903 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:35,903 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. after waiting 0 ms 2023-09-06T20:14:35,903 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:35,903 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:35,904 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1622): Region close journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:35,905 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:14:35,905 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1694031275905"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694031275905"}]},"ts":"1694031275905"} 2023-09-06T20:14:35,906 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:14:35,907 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:14:35,907 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031275907"}]},"ts":"1694031275907"} 2023-09-06T20:14:35,908 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2023-09-06T20:14:35,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=e66eccf326cb10336c74bdb6873cf7b9, ASSIGN}] 2023-09-06T20:14:35,914 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=e66eccf326cb10336c74bdb6873cf7b9, ASSIGN 2023-09-06T20:14:35,915 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e66eccf326cb10336c74bdb6873cf7b9, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33275,1694031217268; forceNewPlan=false, retain=false 2023-09-06T20:14:36,066 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=e66eccf326cb10336c74bdb6873cf7b9, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:36,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:14:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:14:36,222 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:36,223 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7254): Opening region: {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:14:36,223 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:36,223 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:14:36,223 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7296): checking encryption for e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:36,223 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7299): checking classloading for e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:36,224 INFO [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:36,225 INFO [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:14:36,226 INFO [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e66eccf326cb10336c74bdb6873cf7b9 columnFamilyName A 2023-09-06T20:14:36,226 DEBUG [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:14:36,226 INFO [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] regionserver.HStore(324): Store=e66eccf326cb10336c74bdb6873cf7b9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:14:36,226 INFO [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:36,227 INFO [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:14:36,228 INFO [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e66eccf326cb10336c74bdb6873cf7b9 columnFamilyName B 2023-09-06T20:14:36,228 DEBUG [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:14:36,228 INFO [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] regionserver.HStore(324): Store=e66eccf326cb10336c74bdb6873cf7b9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:14:36,228 INFO [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:36,229 INFO [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:14:36,229 INFO [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e66eccf326cb10336c74bdb6873cf7b9 columnFamilyName C 2023-09-06T20:14:36,229 DEBUG [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:14:36,230 INFO [StoreOpener-e66eccf326cb10336c74bdb6873cf7b9-1 {}] regionserver.HStore(324): Store=e66eccf326cb10336c74bdb6873cf7b9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:14:36,230 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:36,230 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:36,231 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:36,232 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-06T20:14:36,232 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1076): writing seq id for e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:36,234 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:14:36,234 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): Opened e66eccf326cb10336c74bdb6873cf7b9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59191678, jitterRate=-0.11797526478767395}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-06T20:14:36,235 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(992): Region open journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:36,235 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., pid=74, masterSystemTime=1694031276219 2023-09-06T20:14:36,237 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:36,237 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:36,237 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=e66eccf326cb10336c74bdb6873cf7b9, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:36,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=74, resume processing ppid=73 2023-09-06T20:14:36,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 in 171 msec 2023-09-06T20:14:36,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=73, resume processing ppid=72 2023-09-06T20:14:36,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=e66eccf326cb10336c74bdb6873cf7b9, ASSIGN in 327 msec 2023-09-06T20:14:36,242 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:14:36,242 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031276242"}]},"ts":"1694031276242"} 2023-09-06T20:14:36,242 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2023-09-06T20:14:36,245 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:14:36,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2023-09-06T20:14:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:14:37,193 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 72 completed 2023-09-06T20:14:37,195 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x279eab79 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:37,200 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43abd882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:37,201 DEBUG [Listener at localhost/35961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:37,203 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48106, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:37,204 DEBUG [Listener at localhost/35961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:14:37,205 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49536, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:14:37,207 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0e83f4c3 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:37,215 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c945309, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:37,217 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x65bbaf59 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:37,221 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@748f7ac1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:37,222 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7c931dab to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:37,229 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@446aa407, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:37,230 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2aba9a6f to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:37,237 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ea0a4d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:37,238 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x70d13a10 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:37,249 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e46bf87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:37,250 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x38d09399 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:37,260 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d1a983, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:37,261 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1f0aa952 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:37,266 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23faad71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:37,267 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x26513133 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:37,272 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdb4c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:37,272 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7b9dc353 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:37,276 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54eabdba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:37,277 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1f084375 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:14:37,281 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a1a9c9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:14:37,298 DEBUG [hconnection-0x1cb15a36-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:37,299 DEBUG [hconnection-0x83839ed-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:37,299 DEBUG [hconnection-0x4df3824f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:37,300 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48120, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:37,300 DEBUG [hconnection-0x22f71bdb-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:37,300 DEBUG [hconnection-0x3aeb29d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:37,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:37,301 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48134, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:37,306 DEBUG [hconnection-0x6b6b92d3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:37,307 DEBUG [hconnection-0x47c74d24-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:37,307 DEBUG [hconnection-0x25557300-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:37,315 DEBUG [hconnection-0x7d916ead-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:37,317 DEBUG [hconnection-0x78b1f0b6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:14:37,317 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48124, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:37,317 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48114, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:37,317 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48154, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:37,317 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48140, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:37,317 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48132, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:37,318 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48158, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=75, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees 2023-09-06T20:14:37,322 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=75, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-06T20:14:37,332 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48182, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:37,333 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=75, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:37,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:37,334 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48170, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:14:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:37,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-06T20:14:37,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:37,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:37,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:37,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:37,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:37,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:37,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/f309c3403aef452a94a309fe1b5f9259 is 50, key is test_row_0/A:col10/1694031277341/Put/seqid=0 2023-09-06T20:14:37,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031337374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031337376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031337378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031337379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031337379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/f309c3403aef452a94a309fe1b5f9259 2023-09-06T20:14:37,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/5fada1f330084a198d229db14b4a7588 is 50, key is test_row_0/B:col10/1694031277341/Put/seqid=0 2023-09-06T20:14:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-06T20:14:37,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/5fada1f330084a198d229db14b4a7588 2023-09-06T20:14:37,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/fd9373a3fcc848dc862a4cab3dca2cf3 is 50, key is test_row_0/C:col10/1694031277341/Put/seqid=0 2023-09-06T20:14:37,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031337481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031337481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031337482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031337483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031337483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/fd9373a3fcc848dc862a4cab3dca2cf3 2023-09-06T20:14:37,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2023-09-06T20:14:37,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:37,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:37,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:37,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(53): pid=76 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:37,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:37,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/f309c3403aef452a94a309fe1b5f9259 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/f309c3403aef452a94a309fe1b5f9259 2023-09-06T20:14:37,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/f309c3403aef452a94a309fe1b5f9259, entries=150, sequenceid=15, filesize=11.7 K 2023-09-06T20:14:37,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/5fada1f330084a198d229db14b4a7588 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/5fada1f330084a198d229db14b4a7588 2023-09-06T20:14:37,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/5fada1f330084a198d229db14b4a7588, entries=150, sequenceid=15, filesize=11.7 K 2023-09-06T20:14:37,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/fd9373a3fcc848dc862a4cab3dca2cf3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/fd9373a3fcc848dc862a4cab3dca2cf3 2023-09-06T20:14:37,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/fd9373a3fcc848dc862a4cab3dca2cf3, entries=150, sequenceid=15, filesize=11.7 K 2023-09-06T20:14:37,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for e66eccf326cb10336c74bdb6873cf7b9 in 159ms, sequenceid=15, compaction requested=false 2023-09-06T20:14:37,510 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2023-09-06T20:14:37,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-06T20:14:37,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2023-09-06T20:14:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:37,645 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-06T20:14:37,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:37,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:37,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:37,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:37,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:37,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:37,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/16ad974fcc9f4e0b95ba68b4066923e0 is 50, key is test_row_0/A:col10/1694031277377/Put/seqid=0 2023-09-06T20:14:37,680 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/16ad974fcc9f4e0b95ba68b4066923e0 2023-09-06T20:14:37,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:37,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/d1df3a825c074283ac982d6973cac546 is 50, key is test_row_0/B:col10/1694031277377/Put/seqid=0 2023-09-06T20:14:37,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031337699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031337700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031337701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031337702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031337702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031337805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031337805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031337805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031337806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031337806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-06T20:14:38,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031338009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031338009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031338009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031338009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031338010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,116 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/d1df3a825c074283ac982d6973cac546 2023-09-06T20:14:38,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/8abfc13fcff14256ba2068fdcc34c90e is 50, key is test_row_0/C:col10/1694031277377/Put/seqid=0 2023-09-06T20:14:38,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031338312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031338312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031338313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031338313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031338313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-06T20:14:38,544 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/8abfc13fcff14256ba2068fdcc34c90e 2023-09-06T20:14:38,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/16ad974fcc9f4e0b95ba68b4066923e0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/16ad974fcc9f4e0b95ba68b4066923e0 2023-09-06T20:14:38,552 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/16ad974fcc9f4e0b95ba68b4066923e0, entries=150, sequenceid=38, filesize=11.7 K 2023-09-06T20:14:38,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/d1df3a825c074283ac982d6973cac546 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d1df3a825c074283ac982d6973cac546 2023-09-06T20:14:38,565 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d1df3a825c074283ac982d6973cac546, entries=150, sequenceid=38, filesize=11.7 K 2023-09-06T20:14:38,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/8abfc13fcff14256ba2068fdcc34c90e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/8abfc13fcff14256ba2068fdcc34c90e 2023-09-06T20:14:38,569 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/8abfc13fcff14256ba2068fdcc34c90e, entries=150, sequenceid=38, filesize=11.7 K 2023-09-06T20:14:38,570 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for e66eccf326cb10336c74bdb6873cf7b9 in 925ms, sequenceid=38, compaction requested=false 2023-09-06T20:14:38,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:38,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:38,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=76 2023-09-06T20:14:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=76 2023-09-06T20:14:38,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=76, resume processing ppid=75 2023-09-06T20:14:38,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=76, ppid=75, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2380 sec 2023-09-06T20:14:38,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees in 1.2740 sec 2023-09-06T20:14:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:38,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-06T20:14:38,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:38,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:38,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:38,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:38,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:38,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:38,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/5784b7790a61475f9eded82518e4537c is 50, key is test_row_0/A:col10/1694031277699/Put/seqid=0 2023-09-06T20:14:38,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031338835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031338836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031338839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031338839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031338840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/5784b7790a61475f9eded82518e4537c 2023-09-06T20:14:38,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/fd11eafa85e5444b98db604377a05a20 is 50, key is test_row_0/B:col10/1694031277699/Put/seqid=0 2023-09-06T20:14:38,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/fd11eafa85e5444b98db604377a05a20 2023-09-06T20:14:38,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/a342a434aa4a4ada89d303b5faf33989 is 50, key is test_row_0/C:col10/1694031277699/Put/seqid=0 2023-09-06T20:14:38,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031338940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031338940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031338942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031338942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:38,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031338944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,031 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-06T20:14:39,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031339143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031339144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031339145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031339146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031339146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/a342a434aa4a4ada89d303b5faf33989 2023-09-06T20:14:39,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/5784b7790a61475f9eded82518e4537c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/5784b7790a61475f9eded82518e4537c 2023-09-06T20:14:39,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/5784b7790a61475f9eded82518e4537c, entries=150, sequenceid=53, filesize=11.7 K 2023-09-06T20:14:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/fd11eafa85e5444b98db604377a05a20 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fd11eafa85e5444b98db604377a05a20 2023-09-06T20:14:39,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fd11eafa85e5444b98db604377a05a20, entries=150, sequenceid=53, filesize=11.7 K 2023-09-06T20:14:39,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/a342a434aa4a4ada89d303b5faf33989 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/a342a434aa4a4ada89d303b5faf33989 2023-09-06T20:14:39,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/a342a434aa4a4ada89d303b5faf33989, entries=150, sequenceid=53, filesize=11.7 K 2023-09-06T20:14:39,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for e66eccf326cb10336c74bdb6873cf7b9 in 491ms, sequenceid=53, compaction requested=true 2023-09-06T20:14:39,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:39,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:39,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:39,307 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:39,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:39,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:39,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:39,307 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:39,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:39,308 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:39,308 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/A is initiating minor compaction (all files) 2023-09-06T20:14:39,308 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/A in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:39,308 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/f309c3403aef452a94a309fe1b5f9259, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/16ad974fcc9f4e0b95ba68b4066923e0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/5784b7790a61475f9eded82518e4537c] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=35.1 K 2023-09-06T20:14:39,308 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:39,308 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/B is initiating minor compaction (all files) 2023-09-06T20:14:39,308 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting f309c3403aef452a94a309fe1b5f9259, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1694031277341 2023-09-06T20:14:39,308 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/B in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:39,308 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/5fada1f330084a198d229db14b4a7588, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d1df3a825c074283ac982d6973cac546, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fd11eafa85e5444b98db604377a05a20] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=35.1 K 2023-09-06T20:14:39,309 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 16ad974fcc9f4e0b95ba68b4066923e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1694031277371 2023-09-06T20:14:39,309 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 5fada1f330084a198d229db14b4a7588, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1694031277341 2023-09-06T20:14:39,309 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5784b7790a61475f9eded82518e4537c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1694031277695 2023-09-06T20:14:39,309 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting d1df3a825c074283ac982d6973cac546, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1694031277371 2023-09-06T20:14:39,310 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting fd11eafa85e5444b98db604377a05a20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1694031277695 2023-09-06T20:14:39,320 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:39,320 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/e98480c4e57842f59c32d8067d84311c is 50, key is test_row_0/B:col10/1694031277699/Put/seqid=0 2023-09-06T20:14:39,321 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#A#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:39,322 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/fbac69b2d0b04fe5a38d57aedff0d94b is 50, key is test_row_0/A:col10/1694031277699/Put/seqid=0 2023-09-06T20:14:39,338 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/e98480c4e57842f59c32d8067d84311c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/e98480c4e57842f59c32d8067d84311c 2023-09-06T20:14:39,341 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/fbac69b2d0b04fe5a38d57aedff0d94b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/fbac69b2d0b04fe5a38d57aedff0d94b 2023-09-06T20:14:39,346 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/B of e66eccf326cb10336c74bdb6873cf7b9 into e98480c4e57842f59c32d8067d84311c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:39,346 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:39,346 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/B, priority=13, startTime=1694031279307; duration=0sec 2023-09-06T20:14:39,346 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:39,346 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:B 2023-09-06T20:14:39,346 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:39,347 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:39,348 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/C is initiating minor compaction (all files) 2023-09-06T20:14:39,348 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/C in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:39,348 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/fd9373a3fcc848dc862a4cab3dca2cf3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/8abfc13fcff14256ba2068fdcc34c90e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/a342a434aa4a4ada89d303b5faf33989] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=35.1 K 2023-09-06T20:14:39,348 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting fd9373a3fcc848dc862a4cab3dca2cf3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1694031277341 2023-09-06T20:14:39,348 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 8abfc13fcff14256ba2068fdcc34c90e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1694031277371 2023-09-06T20:14:39,349 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting a342a434aa4a4ada89d303b5faf33989, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1694031277695 2023-09-06T20:14:39,359 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/A of e66eccf326cb10336c74bdb6873cf7b9 into fbac69b2d0b04fe5a38d57aedff0d94b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:39,359 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:39,359 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/A, priority=13, startTime=1694031279307; duration=0sec 2023-09-06T20:14:39,359 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:39,359 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:A 2023-09-06T20:14:39,367 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#C#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:39,368 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/520045e0bde44a1f81f5e6e0409b0c0a is 50, key is test_row_0/C:col10/1694031277699/Put/seqid=0 2023-09-06T20:14:39,381 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/520045e0bde44a1f81f5e6e0409b0c0a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/520045e0bde44a1f81f5e6e0409b0c0a 2023-09-06T20:14:39,386 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/C of e66eccf326cb10336c74bdb6873cf7b9 into 520045e0bde44a1f81f5e6e0409b0c0a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:39,386 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:39,386 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/C, priority=13, startTime=1694031279307; duration=0sec 2023-09-06T20:14:39,386 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:39,386 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:C 2023-09-06T20:14:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-06T20:14:39,428 INFO [Thread-1447 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 75 completed 2023-09-06T20:14:39,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=77, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees 2023-09-06T20:14:39,432 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=77, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=77 2023-09-06T20:14:39,433 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=77, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:39,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:39,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-06T20:14:39,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:39,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:39,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:39,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:39,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:39,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:39,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/51cb1847abbb4b45a62a3391548d8f96 is 50, key is test_row_0/A:col10/1694031278832/Put/seqid=0 2023-09-06T20:14:39,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031339458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031339459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031339459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031339460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031339460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/51cb1847abbb4b45a62a3391548d8f96 2023-09-06T20:14:39,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/92da2b29796d49568c9ff59afb4b6bdf is 50, key is test_row_0/B:col10/1694031278832/Put/seqid=0 2023-09-06T20:14:39,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/92da2b29796d49568c9ff59afb4b6bdf 2023-09-06T20:14:39,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/de3983ff99ce43b1ab9157c6663c53d7 is 50, key is test_row_0/C:col10/1694031278832/Put/seqid=0 2023-09-06T20:14:39,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/de3983ff99ce43b1ab9157c6663c53d7 2023-09-06T20:14:39,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/51cb1847abbb4b45a62a3391548d8f96 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/51cb1847abbb4b45a62a3391548d8f96 2023-09-06T20:14:39,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/51cb1847abbb4b45a62a3391548d8f96, entries=150, sequenceid=80, filesize=11.7 K 2023-09-06T20:14:39,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/92da2b29796d49568c9ff59afb4b6bdf as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/92da2b29796d49568c9ff59afb4b6bdf 2023-09-06T20:14:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=77 2023-09-06T20:14:39,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/92da2b29796d49568c9ff59afb4b6bdf, entries=150, sequenceid=80, filesize=11.7 K 2023-09-06T20:14:39,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/de3983ff99ce43b1ab9157c6663c53d7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/de3983ff99ce43b1ab9157c6663c53d7 2023-09-06T20:14:39,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/de3983ff99ce43b1ab9157c6663c53d7, entries=150, sequenceid=80, filesize=11.7 K 2023-09-06T20:14:39,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for e66eccf326cb10336c74bdb6873cf7b9 in 104ms, sequenceid=80, compaction requested=false 2023-09-06T20:14:39,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:39,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2023-09-06T20:14:39,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:39,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:39,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:39,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:39,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:39,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:39,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/047f17550e964f5e940799fb0c4fcf82 is 50, key is test_row_0/A:col10/1694031279459/Put/seqid=0 2023-09-06T20:14:39,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/047f17550e964f5e940799fb0c4fcf82 2023-09-06T20:14:39,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-06T20:14:39,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:39,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:39,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:39,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:39,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:39,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/2f6cecef0d104f86bab5ca59097daac7 is 50, key is test_row_0/B:col10/1694031279459/Put/seqid=0 2023-09-06T20:14:39,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031339587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031339588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031339589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031339591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031339593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/2f6cecef0d104f86bab5ca59097daac7 2023-09-06T20:14:39,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/df5ca179b03e4d2c8e11348f7fa1f89c is 50, key is test_row_0/C:col10/1694031279459/Put/seqid=0 2023-09-06T20:14:39,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/df5ca179b03e4d2c8e11348f7fa1f89c 2023-09-06T20:14:39,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/047f17550e964f5e940799fb0c4fcf82 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/047f17550e964f5e940799fb0c4fcf82 2023-09-06T20:14:39,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/047f17550e964f5e940799fb0c4fcf82, entries=250, sequenceid=95, filesize=16.3 K 2023-09-06T20:14:39,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/2f6cecef0d104f86bab5ca59097daac7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/2f6cecef0d104f86bab5ca59097daac7 2023-09-06T20:14:39,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/2f6cecef0d104f86bab5ca59097daac7, entries=150, sequenceid=95, filesize=11.7 K 2023-09-06T20:14:39,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/df5ca179b03e4d2c8e11348f7fa1f89c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/df5ca179b03e4d2c8e11348f7fa1f89c 2023-09-06T20:14:39,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/df5ca179b03e4d2c8e11348f7fa1f89c, entries=150, sequenceid=95, filesize=11.7 K 2023-09-06T20:14:39,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for e66eccf326cb10336c74bdb6873cf7b9 in 93ms, sequenceid=95, compaction requested=true 2023-09-06T20:14:39,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:39,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:39,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:39,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:39,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:39,660 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:39,660 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:39,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:39,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:39,661 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40735 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:39,661 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:39,662 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/A is initiating minor compaction (all files) 2023-09-06T20:14:39,662 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/B is initiating minor compaction (all files) 2023-09-06T20:14:39,662 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/A in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:39,662 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/B in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:39,662 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/fbac69b2d0b04fe5a38d57aedff0d94b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/51cb1847abbb4b45a62a3391548d8f96, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/047f17550e964f5e940799fb0c4fcf82] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=39.8 K 2023-09-06T20:14:39,662 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/e98480c4e57842f59c32d8067d84311c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/92da2b29796d49568c9ff59afb4b6bdf, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/2f6cecef0d104f86bab5ca59097daac7] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=35.2 K 2023-09-06T20:14:39,678 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting e98480c4e57842f59c32d8067d84311c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1694031277695 2023-09-06T20:14:39,679 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting fbac69b2d0b04fe5a38d57aedff0d94b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1694031277695 2023-09-06T20:14:39,679 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 92da2b29796d49568c9ff59afb4b6bdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1694031278832 2023-09-06T20:14:39,679 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 51cb1847abbb4b45a62a3391548d8f96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1694031278832 2023-09-06T20:14:39,679 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 2f6cecef0d104f86bab5ca59097daac7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1694031279459 2023-09-06T20:14:39,679 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 047f17550e964f5e940799fb0c4fcf82, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1694031279458 2023-09-06T20:14:39,687 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:39,687 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/4e4d190ab7a347c5b524131a410e3df8 is 50, key is test_row_0/B:col10/1694031279459/Put/seqid=0 2023-09-06T20:14:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:39,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-06T20:14:39,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:39,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:39,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:39,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:39,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:39,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:39,697 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#A#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:39,698 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/180768c30cee47fda356508cea1e9138 is 50, key is test_row_0/A:col10/1694031279459/Put/seqid=0 2023-09-06T20:14:39,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/1d7029007d5e4606a881a2d47d54e72d is 50, key is test_row_0/A:col10/1694031279590/Put/seqid=0 2023-09-06T20:14:39,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031339703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031339704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031339705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031339707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031339708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,722 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/4e4d190ab7a347c5b524131a410e3df8 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/4e4d190ab7a347c5b524131a410e3df8 2023-09-06T20:14:39,726 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/B of e66eccf326cb10336c74bdb6873cf7b9 into 4e4d190ab7a347c5b524131a410e3df8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:39,726 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:39,726 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/B, priority=13, startTime=1694031279660; duration=0sec 2023-09-06T20:14:39,726 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:39,726 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:B 2023-09-06T20:14:39,726 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:39,727 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:39,727 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/C is initiating minor compaction (all files) 2023-09-06T20:14:39,728 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/C in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:39,728 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/520045e0bde44a1f81f5e6e0409b0c0a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/de3983ff99ce43b1ab9157c6663c53d7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/df5ca179b03e4d2c8e11348f7fa1f89c] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=35.2 K 2023-09-06T20:14:39,728 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 520045e0bde44a1f81f5e6e0409b0c0a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1694031277695 2023-09-06T20:14:39,728 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting de3983ff99ce43b1ab9157c6663c53d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1694031278832 2023-09-06T20:14:39,729 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting df5ca179b03e4d2c8e11348f7fa1f89c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1694031279459 2023-09-06T20:14:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=77 2023-09-06T20:14:39,738 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#C#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:39,739 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/cc22c4bccfa2432d883f0961d0c88ed3 is 50, key is test_row_0/C:col10/1694031279459/Put/seqid=0 2023-09-06T20:14:39,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-06T20:14:39,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:39,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:39,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:39,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:39,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:39,773 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/cc22c4bccfa2432d883f0961d0c88ed3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/cc22c4bccfa2432d883f0961d0c88ed3 2023-09-06T20:14:39,778 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/C of e66eccf326cb10336c74bdb6873cf7b9 into cc22c4bccfa2432d883f0961d0c88ed3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:39,778 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:39,778 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/C, priority=13, startTime=1694031279660; duration=0sec 2023-09-06T20:14:39,778 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:39,778 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:C 2023-09-06T20:14:39,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031339812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031339812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031339814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031339815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031339815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:39,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-06T20:14:39,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:39,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:39,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:39,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:39,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031340014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031340015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031340017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031340019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031340019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=77 2023-09-06T20:14:40,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-06T20:14:40,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:40,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:40,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:40,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/1d7029007d5e4606a881a2d47d54e72d 2023-09-06T20:14:40,121 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/180768c30cee47fda356508cea1e9138 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/180768c30cee47fda356508cea1e9138 2023-09-06T20:14:40,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/02113d7581d14c3db9a525dd0d451aa7 is 50, key is test_row_0/B:col10/1694031279590/Put/seqid=0 2023-09-06T20:14:40,131 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/A of e66eccf326cb10336c74bdb6873cf7b9 into 180768c30cee47fda356508cea1e9138(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:40,131 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:40,131 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/A, priority=13, startTime=1694031279660; duration=0sec 2023-09-06T20:14:40,131 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:40,131 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:A 2023-09-06T20:14:40,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-06T20:14:40,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:40,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:40,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:40,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031340319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031340320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031340322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031340323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031340325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-06T20:14:40,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:40,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:40,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:40,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-06T20:14:40,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:40,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:40,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:40,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=77 2023-09-06T20:14:40,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/02113d7581d14c3db9a525dd0d451aa7 2023-09-06T20:14:40,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/6fa386f2d22d4dae802d9823e36cada7 is 50, key is test_row_0/C:col10/1694031279590/Put/seqid=0 2023-09-06T20:14:40,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-06T20:14:40,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:40,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:40,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:40,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-06T20:14:40,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:40,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:40,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:40,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031340823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031340826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031340827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031340828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031340833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:40,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/6fa386f2d22d4dae802d9823e36cada7 2023-09-06T20:14:40,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-06T20:14:40,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:40,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:40,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:40,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:40,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/1d7029007d5e4606a881a2d47d54e72d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/1d7029007d5e4606a881a2d47d54e72d 2023-09-06T20:14:40,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/1d7029007d5e4606a881a2d47d54e72d, entries=200, sequenceid=119, filesize=14.0 K 2023-09-06T20:14:40,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/02113d7581d14c3db9a525dd0d451aa7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/02113d7581d14c3db9a525dd0d451aa7 2023-09-06T20:14:40,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/02113d7581d14c3db9a525dd0d451aa7, entries=150, sequenceid=119, filesize=11.7 K 2023-09-06T20:14:40,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/6fa386f2d22d4dae802d9823e36cada7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/6fa386f2d22d4dae802d9823e36cada7 2023-09-06T20:14:40,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/6fa386f2d22d4dae802d9823e36cada7, entries=150, sequenceid=119, filesize=11.7 K 2023-09-06T20:14:40,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for e66eccf326cb10336c74bdb6873cf7b9 in 1293ms, sequenceid=119, compaction requested=false 2023-09-06T20:14:40,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:41,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-06T20:14:41,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:41,124 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-06T20:14:41,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:41,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:41,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:41,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:41,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:41,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:41,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/34d242df3f954b5daf7dbbc302d137e5 is 50, key is test_row_0/A:col10/1694031279702/Put/seqid=0 2023-09-06T20:14:41,140 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/34d242df3f954b5daf7dbbc302d137e5 2023-09-06T20:14:41,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/84f5ef8fd35943278f0d06dae1838e7c is 50, key is test_row_0/B:col10/1694031279702/Put/seqid=0 2023-09-06T20:14:41,176 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/84f5ef8fd35943278f0d06dae1838e7c 2023-09-06T20:14:41,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/0b04dde83edf4925876026457bb13749 is 50, key is test_row_0/C:col10/1694031279702/Put/seqid=0 2023-09-06T20:14:41,192 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/0b04dde83edf4925876026457bb13749 2023-09-06T20:14:41,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/34d242df3f954b5daf7dbbc302d137e5 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/34d242df3f954b5daf7dbbc302d137e5 2023-09-06T20:14:41,201 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/34d242df3f954b5daf7dbbc302d137e5, entries=150, sequenceid=135, filesize=11.8 K 2023-09-06T20:14:41,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/84f5ef8fd35943278f0d06dae1838e7c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/84f5ef8fd35943278f0d06dae1838e7c 2023-09-06T20:14:41,214 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/84f5ef8fd35943278f0d06dae1838e7c, entries=150, sequenceid=135, filesize=11.8 K 2023-09-06T20:14:41,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/0b04dde83edf4925876026457bb13749 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/0b04dde83edf4925876026457bb13749 2023-09-06T20:14:41,219 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/0b04dde83edf4925876026457bb13749, entries=150, sequenceid=135, filesize=11.8 K 2023-09-06T20:14:41,220 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for e66eccf326cb10336c74bdb6873cf7b9 in 95ms, sequenceid=135, compaction requested=true 2023-09-06T20:14:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=78 2023-09-06T20:14:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=78 2023-09-06T20:14:41,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=78, resume processing ppid=77 2023-09-06T20:14:41,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=78, ppid=77, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7880 sec 2023-09-06T20:14:41,223 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=77, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees in 1.7930 sec 2023-09-06T20:14:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=77 2023-09-06T20:14:41,536 INFO [Thread-1447 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 77 completed 2023-09-06T20:14:41,538 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=79, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees 2023-09-06T20:14:41,540 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=79, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:14:41,541 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=79, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:41,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:14:41,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2023-09-06T20:14:41,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:41,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:41,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:41,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=80 2023-09-06T20:14:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=80 2023-09-06T20:14:41,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=80, resume processing ppid=79 2023-09-06T20:14:41,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=80, ppid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 159 msec 2023-09-06T20:14:41,703 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees in 164 msec 2023-09-06T20:14:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:14:41,843 INFO [Thread-1447 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 79 completed 2023-09-06T20:14:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:41,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:14:41,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:41,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:41,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:41,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:41,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:41,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:41,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees 2023-09-06T20:14:41,846 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:14:41,853 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=81, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:41,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:41,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/234fa368a45347f3890cceb871bb8134 is 50, key is test_row_0/A:col10/1694031281841/Put/seqid=0 2023-09-06T20:14:41,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031341870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:41,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031341871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:41,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031341871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:41,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031341871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:41,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031341872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:14:41,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031341974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:41,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031341974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:41,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031341975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:41,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031341975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:41,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031341982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2023-09-06T20:14:42,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:42,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:42,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:42,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(53): pid=82 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,130 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 46c263e9fd401abca49830e3d9e96c8e changed from -1.0 to 0.0, refreshing cache 2023-09-06T20:14:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:14:42,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2023-09-06T20:14:42,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:42,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:42,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:42,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(53): pid=82 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031342176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031342176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031342177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031342179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031342186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/234fa368a45347f3890cceb871bb8134 2023-09-06T20:14:42,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/99a7ffa5a22b4a70bf8d5e05c6ddf8af is 50, key is test_row_0/B:col10/1694031281841/Put/seqid=0 2023-09-06T20:14:42,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/99a7ffa5a22b4a70bf8d5e05c6ddf8af 2023-09-06T20:14:42,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/960104e7e85e49ffb4217aa9348722c6 is 50, key is test_row_0/C:col10/1694031281841/Put/seqid=0 2023-09-06T20:14:42,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2023-09-06T20:14:42,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:42,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(53): pid=82 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:14:42,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2023-09-06T20:14:42,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:42,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:42,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:42,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(53): pid=82 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031342479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031342479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031342482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031342483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031342489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2023-09-06T20:14:42,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:42,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:42,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:42,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(53): pid=82 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:42,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/960104e7e85e49ffb4217aa9348722c6 2023-09-06T20:14:42,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/234fa368a45347f3890cceb871bb8134 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/234fa368a45347f3890cceb871bb8134 2023-09-06T20:14:42,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/234fa368a45347f3890cceb871bb8134, entries=150, sequenceid=148, filesize=11.8 K 2023-09-06T20:14:42,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/99a7ffa5a22b4a70bf8d5e05c6ddf8af as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/99a7ffa5a22b4a70bf8d5e05c6ddf8af 2023-09-06T20:14:42,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/99a7ffa5a22b4a70bf8d5e05c6ddf8af, entries=150, sequenceid=148, filesize=11.8 K 2023-09-06T20:14:42,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/960104e7e85e49ffb4217aa9348722c6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/960104e7e85e49ffb4217aa9348722c6 2023-09-06T20:14:42,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/960104e7e85e49ffb4217aa9348722c6, entries=150, sequenceid=148, filesize=11.8 K 2023-09-06T20:14:42,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e66eccf326cb10336c74bdb6873cf7b9 in 882ms, sequenceid=148, compaction requested=true 2023-09-06T20:14:42,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:42,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:42,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:42,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:42,726 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:42,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:42,726 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:42,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:42,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:42,727 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:42,727 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:42,727 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/A is initiating minor compaction (all files) 2023-09-06T20:14:42,727 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/B is initiating minor compaction (all files) 2023-09-06T20:14:42,727 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/A in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:42,727 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/B in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:42,727 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/180768c30cee47fda356508cea1e9138, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/1d7029007d5e4606a881a2d47d54e72d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/34d242df3f954b5daf7dbbc302d137e5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/234fa368a45347f3890cceb871bb8134] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=49.5 K 2023-09-06T20:14:42,727 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/4e4d190ab7a347c5b524131a410e3df8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/02113d7581d14c3db9a525dd0d451aa7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/84f5ef8fd35943278f0d06dae1838e7c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/99a7ffa5a22b4a70bf8d5e05c6ddf8af] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=47.2 K 2023-09-06T20:14:42,729 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 180768c30cee47fda356508cea1e9138, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1694031279459 2023-09-06T20:14:42,730 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 4e4d190ab7a347c5b524131a410e3df8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1694031279459 2023-09-06T20:14:42,730 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1d7029007d5e4606a881a2d47d54e72d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1694031279589 2023-09-06T20:14:42,730 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 02113d7581d14c3db9a525dd0d451aa7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1694031279590 2023-09-06T20:14:42,730 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 34d242df3f954b5daf7dbbc302d137e5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1694031279701 2023-09-06T20:14:42,732 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 84f5ef8fd35943278f0d06dae1838e7c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1694031279701 2023-09-06T20:14:42,732 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 234fa368a45347f3890cceb871bb8134, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1694031281839 2023-09-06T20:14:42,733 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 99a7ffa5a22b4a70bf8d5e05c6ddf8af, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1694031281839 2023-09-06T20:14:42,742 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:42,743 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/f0b7ad9996a64bc199f180d58a9bc84c is 50, key is test_row_0/A:col10/1694031281841/Put/seqid=0 2023-09-06T20:14:42,744 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#B#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:42,745 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/40422d6b3dd049239053f289b8a6b993 is 50, key is test_row_0/B:col10/1694031281841/Put/seqid=0 2023-09-06T20:14:42,754 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/f0b7ad9996a64bc199f180d58a9bc84c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/f0b7ad9996a64bc199f180d58a9bc84c 2023-09-06T20:14:42,759 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/A of e66eccf326cb10336c74bdb6873cf7b9 into f0b7ad9996a64bc199f180d58a9bc84c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:42,759 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:42,759 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/A, priority=12, startTime=1694031282726; duration=0sec 2023-09-06T20:14:42,759 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:42,759 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:A 2023-09-06T20:14:42,759 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:42,761 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:42,761 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/C is initiating minor compaction (all files) 2023-09-06T20:14:42,761 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/C in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:42,761 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/cc22c4bccfa2432d883f0961d0c88ed3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/6fa386f2d22d4dae802d9823e36cada7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/0b04dde83edf4925876026457bb13749, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/960104e7e85e49ffb4217aa9348722c6] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=47.2 K 2023-09-06T20:14:42,762 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting cc22c4bccfa2432d883f0961d0c88ed3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1694031279459 2023-09-06T20:14:42,762 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6fa386f2d22d4dae802d9823e36cada7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1694031279590 2023-09-06T20:14:42,762 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 0b04dde83edf4925876026457bb13749, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1694031279701 2023-09-06T20:14:42,763 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 960104e7e85e49ffb4217aa9348722c6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1694031281839 2023-09-06T20:14:42,775 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:42,775 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/61f5c45694c54789a0c182c2c1a7fec1 is 50, key is test_row_0/C:col10/1694031281841/Put/seqid=0 2023-09-06T20:14:42,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2023-09-06T20:14:42,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:42,780 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-06T20:14:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:42,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/b2fb56c9b9134e23a622292e6f2b74ab is 50, key is test_row_0/A:col10/1694031281871/Put/seqid=0 2023-09-06T20:14:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:14:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:42,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:42,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031342992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031342993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031342993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031342994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:42,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031342994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:43,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031343096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:43,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031343096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031343097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031343097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:43,159 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/40422d6b3dd049239053f289b8a6b993 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/40422d6b3dd049239053f289b8a6b993 2023-09-06T20:14:43,165 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/B of e66eccf326cb10336c74bdb6873cf7b9 into 40422d6b3dd049239053f289b8a6b993(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:43,165 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:43,165 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/B, priority=12, startTime=1694031282726; duration=0sec 2023-09-06T20:14:43,165 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:43,165 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:B 2023-09-06T20:14:43,184 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/61f5c45694c54789a0c182c2c1a7fec1 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/61f5c45694c54789a0c182c2c1a7fec1 2023-09-06T20:14:43,188 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/b2fb56c9b9134e23a622292e6f2b74ab 2023-09-06T20:14:43,189 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/C of e66eccf326cb10336c74bdb6873cf7b9 into 61f5c45694c54789a0c182c2c1a7fec1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:43,189 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:43,189 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/C, priority=12, startTime=1694031282726; duration=0sec 2023-09-06T20:14:43,189 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:43,189 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:C 2023-09-06T20:14:43,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/51a3189079aa4d0ca3e3054885e5627e is 50, key is test_row_0/B:col10/1694031281871/Put/seqid=0 2023-09-06T20:14:43,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031343298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:43,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031343299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:43,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031343301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:43,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031343301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:43,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031343601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:43,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031343603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:43,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031343604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:43,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031343605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:43,610 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/51a3189079aa4d0ca3e3054885e5627e 2023-09-06T20:14:43,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/2e239b5d1ab040899619d5a3dc3245a9 is 50, key is test_row_0/C:col10/1694031281871/Put/seqid=0 2023-09-06T20:14:43,624 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/2e239b5d1ab040899619d5a3dc3245a9 2023-09-06T20:14:43,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/b2fb56c9b9134e23a622292e6f2b74ab as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b2fb56c9b9134e23a622292e6f2b74ab 2023-09-06T20:14:43,634 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b2fb56c9b9134e23a622292e6f2b74ab, entries=150, sequenceid=173, filesize=11.8 K 2023-09-06T20:14:43,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/51a3189079aa4d0ca3e3054885e5627e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/51a3189079aa4d0ca3e3054885e5627e 2023-09-06T20:14:43,638 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/51a3189079aa4d0ca3e3054885e5627e, entries=150, sequenceid=173, filesize=11.8 K 2023-09-06T20:14:43,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/2e239b5d1ab040899619d5a3dc3245a9 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/2e239b5d1ab040899619d5a3dc3245a9 2023-09-06T20:14:43,644 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/2e239b5d1ab040899619d5a3dc3245a9, entries=150, sequenceid=173, filesize=11.8 K 2023-09-06T20:14:43,644 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e66eccf326cb10336c74bdb6873cf7b9 in 864ms, sequenceid=173, compaction requested=false 2023-09-06T20:14:43,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:43,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:43,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=82 2023-09-06T20:14:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=82 2023-09-06T20:14:43,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=82, resume processing ppid=81 2023-09-06T20:14:43,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=82, ppid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7920 sec 2023-09-06T20:14:43,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees in 1.8030 sec 2023-09-06T20:14:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:14:43,956 INFO [Thread-1447 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 81 completed 2023-09-06T20:14:43,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees 2023-09-06T20:14:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=83 2023-09-06T20:14:43,960 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:43,961 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=83, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:43,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:44,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-06T20:14:44,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:44,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:44,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:44,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:44,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:44,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:44,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/d2fbf02e84144afc9801263339539afd is 50, key is test_row_0/A:col10/1694031282989/Put/seqid=0 2023-09-06T20:14:44,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031344053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=83 2023-09-06T20:14:44,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031344107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:44,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031344107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:44,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031344110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:44,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2023-09-06T20:14:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031344112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(53): pid=84 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031344156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=83 2023-09-06T20:14:44,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2023-09-06T20:14:44,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:44,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(53): pid=84 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031344359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:44,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2023-09-06T20:14:44,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:44,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(53): pid=84 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/d2fbf02e84144afc9801263339539afd 2023-09-06T20:14:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/a78d2292e5d5408e86452894850b1649 is 50, key is test_row_0/B:col10/1694031282989/Put/seqid=0 2023-09-06T20:14:44,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/a78d2292e5d5408e86452894850b1649 2023-09-06T20:14:44,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/1429b807c3cf491080c1feb322947354 is 50, key is test_row_0/C:col10/1694031282989/Put/seqid=0 2023-09-06T20:14:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=83 2023-09-06T20:14:44,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2023-09-06T20:14:44,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:44,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(53): pid=84 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031344662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:44,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2023-09-06T20:14:44,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:44,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(53): pid=84 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:44,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/1429b807c3cf491080c1feb322947354 2023-09-06T20:14:44,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/d2fbf02e84144afc9801263339539afd as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d2fbf02e84144afc9801263339539afd 2023-09-06T20:14:44,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d2fbf02e84144afc9801263339539afd, entries=150, sequenceid=188, filesize=11.8 K 2023-09-06T20:14:44,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/a78d2292e5d5408e86452894850b1649 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a78d2292e5d5408e86452894850b1649 2023-09-06T20:14:44,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a78d2292e5d5408e86452894850b1649, entries=150, sequenceid=188, filesize=11.8 K 2023-09-06T20:14:44,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/1429b807c3cf491080c1feb322947354 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/1429b807c3cf491080c1feb322947354 2023-09-06T20:14:44,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/1429b807c3cf491080c1feb322947354, entries=150, sequenceid=188, filesize=11.8 K 2023-09-06T20:14:44,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e66eccf326cb10336c74bdb6873cf7b9 in 865ms, sequenceid=188, compaction requested=true 2023-09-06T20:14:44,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:44,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:44,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:44,870 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:44,870 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:44,870 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:44,871 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/A is initiating minor compaction (all files) 2023-09-06T20:14:44,871 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:44,871 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/A in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,871 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/B is initiating minor compaction (all files) 2023-09-06T20:14:44,871 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/f0b7ad9996a64bc199f180d58a9bc84c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b2fb56c9b9134e23a622292e6f2b74ab, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d2fbf02e84144afc9801263339539afd] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=35.9 K 2023-09-06T20:14:44,871 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/B in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,871 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/40422d6b3dd049239053f289b8a6b993, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/51a3189079aa4d0ca3e3054885e5627e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a78d2292e5d5408e86452894850b1649] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=35.9 K 2023-09-06T20:14:44,872 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting f0b7ad9996a64bc199f180d58a9bc84c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1694031281839 2023-09-06T20:14:44,872 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 40422d6b3dd049239053f289b8a6b993, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1694031281839 2023-09-06T20:14:44,872 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting b2fb56c9b9134e23a622292e6f2b74ab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1694031281865 2023-09-06T20:14:44,872 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 51a3189079aa4d0ca3e3054885e5627e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1694031281865 2023-09-06T20:14:44,872 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting d2fbf02e84144afc9801263339539afd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1694031282989 2023-09-06T20:14:44,873 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting a78d2292e5d5408e86452894850b1649, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1694031282989 2023-09-06T20:14:44,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2023-09-06T20:14:44,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,880 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-06T20:14:44,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:44,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:44,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:44,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:44,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:44,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:44,882 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:44,882 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#B#compaction#260 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:44,883 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/3d68f96319a146a6be47aaa34cd1d559 is 50, key is test_row_0/A:col10/1694031282989/Put/seqid=0 2023-09-06T20:14:44,883 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/08512a68ea644f2ca78f56f0a5166162 is 50, key is test_row_0/B:col10/1694031282989/Put/seqid=0 2023-09-06T20:14:44,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/53f4210e0c9840a097c471f5c41a527d is 50, key is test_row_0/A:col10/1694031284050/Put/seqid=0 2023-09-06T20:14:44,890 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/53f4210e0c9840a097c471f5c41a527d 2023-09-06T20:14:44,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/c8d850515ede473eae6eef0d0e61648f is 50, key is test_row_0/B:col10/1694031284050/Put/seqid=0 2023-09-06T20:14:44,901 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/c8d850515ede473eae6eef0d0e61648f 2023-09-06T20:14:44,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/ddab64a23b3a43fe9cb0730b421c5ffe is 50, key is test_row_0/C:col10/1694031284050/Put/seqid=0 2023-09-06T20:14:44,918 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/3d68f96319a146a6be47aaa34cd1d559 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/3d68f96319a146a6be47aaa34cd1d559 2023-09-06T20:14:44,920 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/ddab64a23b3a43fe9cb0730b421c5ffe 2023-09-06T20:14:44,925 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/A of e66eccf326cb10336c74bdb6873cf7b9 into 3d68f96319a146a6be47aaa34cd1d559(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:44,925 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:44,925 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/A, priority=13, startTime=1694031284869; duration=0sec 2023-09-06T20:14:44,925 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:44,925 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:A 2023-09-06T20:14:44,925 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:44,927 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:44,927 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/C is initiating minor compaction (all files) 2023-09-06T20:14:44,927 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/C in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,927 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/61f5c45694c54789a0c182c2c1a7fec1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/2e239b5d1ab040899619d5a3dc3245a9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/1429b807c3cf491080c1feb322947354] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=35.9 K 2023-09-06T20:14:44,927 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 61f5c45694c54789a0c182c2c1a7fec1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1694031281839 2023-09-06T20:14:44,928 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2e239b5d1ab040899619d5a3dc3245a9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1694031281865 2023-09-06T20:14:44,930 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1429b807c3cf491080c1feb322947354, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1694031282989 2023-09-06T20:14:44,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/53f4210e0c9840a097c471f5c41a527d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/53f4210e0c9840a097c471f5c41a527d 2023-09-06T20:14:44,938 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#C#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:44,939 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/d31fc018e1964bf8b26bdd55dff8155b is 50, key is test_row_0/C:col10/1694031282989/Put/seqid=0 2023-09-06T20:14:44,941 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/53f4210e0c9840a097c471f5c41a527d, entries=150, sequenceid=211, filesize=11.8 K 2023-09-06T20:14:44,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/c8d850515ede473eae6eef0d0e61648f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/c8d850515ede473eae6eef0d0e61648f 2023-09-06T20:14:44,947 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/c8d850515ede473eae6eef0d0e61648f, entries=150, sequenceid=211, filesize=11.8 K 2023-09-06T20:14:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/ddab64a23b3a43fe9cb0730b421c5ffe as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/ddab64a23b3a43fe9cb0730b421c5ffe 2023-09-06T20:14:44,955 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/ddab64a23b3a43fe9cb0730b421c5ffe, entries=150, sequenceid=211, filesize=11.8 K 2023-09-06T20:14:44,956 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for e66eccf326cb10336c74bdb6873cf7b9 in 76ms, sequenceid=211, compaction requested=false 2023-09-06T20:14:44,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:44,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:44,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=84 2023-09-06T20:14:44,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=84 2023-09-06T20:14:44,957 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/d31fc018e1964bf8b26bdd55dff8155b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/d31fc018e1964bf8b26bdd55dff8155b 2023-09-06T20:14:44,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=84, resume processing ppid=83 2023-09-06T20:14:44,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=84, ppid=83, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 996 msec 2023-09-06T20:14:44,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=83, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees in 1.0020 sec 2023-09-06T20:14:44,963 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/C of e66eccf326cb10336c74bdb6873cf7b9 into d31fc018e1964bf8b26bdd55dff8155b(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:44,963 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:44,963 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/C, priority=13, startTime=1694031284870; duration=0sec 2023-09-06T20:14:44,963 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:44,963 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:C 2023-09-06T20:14:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=83 2023-09-06T20:14:45,064 INFO [Thread-1447 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 83 completed 2023-09-06T20:14:45,065 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2023-09-06T20:14:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-06T20:14:45,068 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:45,069 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:45,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:45,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:45,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:14:45,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:45,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:45,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:45,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:45,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:45,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:45,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/7d744ffdb22e4656bd144b5bdc7aabe3 is 50, key is test_row_0/A:col10/1694031285118/Put/seqid=0 2023-09-06T20:14:45,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031345147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031345147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031345147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031345148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031345167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-06T20:14:45,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2023-09-06T20:14:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:45,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031345251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031345251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031345251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031345252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,293 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/08512a68ea644f2ca78f56f0a5166162 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/08512a68ea644f2ca78f56f0a5166162 2023-09-06T20:14:45,298 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/B of e66eccf326cb10336c74bdb6873cf7b9 into 08512a68ea644f2ca78f56f0a5166162(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:45,298 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:45,298 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/B, priority=13, startTime=1694031284869; duration=0sec 2023-09-06T20:14:45,298 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:45,298 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:B 2023-09-06T20:14:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-06T20:14:45,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2023-09-06T20:14:45,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:45,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:45,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:45,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031345454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031345454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031345454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031345455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2023-09-06T20:14:45,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:45,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:45,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:45,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/7d744ffdb22e4656bd144b5bdc7aabe3 2023-09-06T20:14:45,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/8bf81faf82d04229853e9bcacd5d4286 is 50, key is test_row_0/B:col10/1694031285118/Put/seqid=0 2023-09-06T20:14:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-06T20:14:45,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2023-09-06T20:14:45,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:45,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:45,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:45,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031345756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031345758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031345759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031345759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:45,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2023-09-06T20:14:45,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:45,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:45,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:45,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/8bf81faf82d04229853e9bcacd5d4286 2023-09-06T20:14:45,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/79e97478a7224acdbdf222d47963e827 is 50, key is test_row_0/C:col10/1694031285118/Put/seqid=0 2023-09-06T20:14:45,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2023-09-06T20:14:45,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:45,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:45,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:45,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:46,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2023-09-06T20:14:46,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:46,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:46,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:46,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:46,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-06T20:14:46,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031346178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:46,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031346259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:46,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031346263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:46,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031346264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:46,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031346265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:46,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2023-09-06T20:14:46,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:46,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:46,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:46,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:46,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:46,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/79e97478a7224acdbdf222d47963e827 2023-09-06T20:14:46,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/7d744ffdb22e4656bd144b5bdc7aabe3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/7d744ffdb22e4656bd144b5bdc7aabe3 2023-09-06T20:14:46,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/7d744ffdb22e4656bd144b5bdc7aabe3, entries=150, sequenceid=226, filesize=11.8 K 2023-09-06T20:14:46,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/8bf81faf82d04229853e9bcacd5d4286 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8bf81faf82d04229853e9bcacd5d4286 2023-09-06T20:14:46,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8bf81faf82d04229853e9bcacd5d4286, entries=150, sequenceid=226, filesize=11.8 K 2023-09-06T20:14:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/79e97478a7224acdbdf222d47963e827 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/79e97478a7224acdbdf222d47963e827 2023-09-06T20:14:46,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/79e97478a7224acdbdf222d47963e827, entries=150, sequenceid=226, filesize=11.8 K 2023-09-06T20:14:46,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e66eccf326cb10336c74bdb6873cf7b9 in 1266ms, sequenceid=226, compaction requested=true 2023-09-06T20:14:46,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:46,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:46,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:46,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:46,388 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:46,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:46,388 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:46,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:46,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:46,389 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:46,389 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/A is initiating minor compaction (all files) 2023-09-06T20:14:46,389 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:46,389 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/A in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:46,389 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/B is initiating minor compaction (all files) 2023-09-06T20:14:46,389 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/3d68f96319a146a6be47aaa34cd1d559, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/53f4210e0c9840a097c471f5c41a527d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/7d744ffdb22e4656bd144b5bdc7aabe3] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=36.0 K 2023-09-06T20:14:46,389 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/B in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:46,389 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/08512a68ea644f2ca78f56f0a5166162, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/c8d850515ede473eae6eef0d0e61648f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8bf81faf82d04229853e9bcacd5d4286] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=36.0 K 2023-09-06T20:14:46,390 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3d68f96319a146a6be47aaa34cd1d559, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1694031282989 2023-09-06T20:14:46,390 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 08512a68ea644f2ca78f56f0a5166162, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1694031282989 2023-09-06T20:14:46,390 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 53f4210e0c9840a097c471f5c41a527d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1694031284042 2023-09-06T20:14:46,390 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting c8d850515ede473eae6eef0d0e61648f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1694031284042 2023-09-06T20:14:46,391 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7d744ffdb22e4656bd144b5bdc7aabe3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1694031285118 2023-09-06T20:14:46,391 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 8bf81faf82d04229853e9bcacd5d4286, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1694031285118 2023-09-06T20:14:46,397 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:46,397 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#B#compaction#269 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:46,397 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/036d435a627c48768b7e5e99466ff638 is 50, key is test_row_0/A:col10/1694031285118/Put/seqid=0 2023-09-06T20:14:46,398 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/58252bd410ca413da28d725b7e13b98d is 50, key is test_row_0/B:col10/1694031285118/Put/seqid=0 2023-09-06T20:14:46,408 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/036d435a627c48768b7e5e99466ff638 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/036d435a627c48768b7e5e99466ff638 2023-09-06T20:14:46,419 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/58252bd410ca413da28d725b7e13b98d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/58252bd410ca413da28d725b7e13b98d 2023-09-06T20:14:46,424 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/B of e66eccf326cb10336c74bdb6873cf7b9 into 58252bd410ca413da28d725b7e13b98d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:46,424 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:46,424 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/B, priority=13, startTime=1694031286388; duration=0sec 2023-09-06T20:14:46,425 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/A of e66eccf326cb10336c74bdb6873cf7b9 into 036d435a627c48768b7e5e99466ff638(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:46,425 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:46,425 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:46,425 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:B 2023-09-06T20:14:46,425 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/A, priority=13, startTime=1694031286388; duration=0sec 2023-09-06T20:14:46,425 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:46,425 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:46,425 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:A 2023-09-06T20:14:46,426 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:46,426 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/C is initiating minor compaction (all files) 2023-09-06T20:14:46,426 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/C in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:46,426 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/d31fc018e1964bf8b26bdd55dff8155b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/ddab64a23b3a43fe9cb0730b421c5ffe, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/79e97478a7224acdbdf222d47963e827] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=36.0 K 2023-09-06T20:14:46,426 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting d31fc018e1964bf8b26bdd55dff8155b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1694031282989 2023-09-06T20:14:46,427 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting ddab64a23b3a43fe9cb0730b421c5ffe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1694031284042 2023-09-06T20:14:46,427 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 79e97478a7224acdbdf222d47963e827, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1694031285118 2023-09-06T20:14:46,433 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#C#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:46,433 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/25355f9d38de42fc8d95dc63e2af23e8 is 50, key is test_row_0/C:col10/1694031285118/Put/seqid=0 2023-09-06T20:14:46,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2023-09-06T20:14:46,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:46,445 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-06T20:14:46,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:46,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:46,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:46,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:46,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:46,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:46,446 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/25355f9d38de42fc8d95dc63e2af23e8 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/25355f9d38de42fc8d95dc63e2af23e8 2023-09-06T20:14:46,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/d124867072cd4a2b81ca098570ddaee4 is 50, key is test_row_0/A:col10/1694031285147/Put/seqid=0 2023-09-06T20:14:46,450 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/C of e66eccf326cb10336c74bdb6873cf7b9 into 25355f9d38de42fc8d95dc63e2af23e8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:46,450 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:46,450 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/C, priority=13, startTime=1694031286388; duration=0sec 2023-09-06T20:14:46,450 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:46,450 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:C 2023-09-06T20:14:46,856 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/d124867072cd4a2b81ca098570ddaee4 2023-09-06T20:14:46,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/36cafd16d7804eecbdb33e54aea7fc26 is 50, key is test_row_0/B:col10/1694031285147/Put/seqid=0 2023-09-06T20:14:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-06T20:14:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:47,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:47,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031347277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031347279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031347280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031347280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,288 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/36cafd16d7804eecbdb33e54aea7fc26 2023-09-06T20:14:47,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/ae46ee0300dd41f6b35f370b3c105699 is 50, key is test_row_0/C:col10/1694031285147/Put/seqid=0 2023-09-06T20:14:47,300 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/ae46ee0300dd41f6b35f370b3c105699 2023-09-06T20:14:47,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/d124867072cd4a2b81ca098570ddaee4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d124867072cd4a2b81ca098570ddaee4 2023-09-06T20:14:47,309 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d124867072cd4a2b81ca098570ddaee4, entries=150, sequenceid=252, filesize=11.8 K 2023-09-06T20:14:47,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/36cafd16d7804eecbdb33e54aea7fc26 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/36cafd16d7804eecbdb33e54aea7fc26 2023-09-06T20:14:47,313 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/36cafd16d7804eecbdb33e54aea7fc26, entries=150, sequenceid=252, filesize=11.8 K 2023-09-06T20:14:47,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/ae46ee0300dd41f6b35f370b3c105699 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/ae46ee0300dd41f6b35f370b3c105699 2023-09-06T20:14:47,318 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/ae46ee0300dd41f6b35f370b3c105699, entries=150, sequenceid=252, filesize=11.8 K 2023-09-06T20:14:47,319 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for e66eccf326cb10336c74bdb6873cf7b9 in 875ms, sequenceid=252, compaction requested=false 2023-09-06T20:14:47,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:47,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:47,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=86 2023-09-06T20:14:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=86 2023-09-06T20:14:47,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=86, resume processing ppid=85 2023-09-06T20:14:47,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=86, ppid=85, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2510 sec 2023-09-06T20:14:47,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=85, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 2.2580 sec 2023-09-06T20:14:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:47,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-06T20:14:47,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:47,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:47,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:47,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:47,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:47,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:47,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/1080f9fc074d4780bd7742086675d2d3 is 50, key is test_row_0/A:col10/1694031287279/Put/seqid=0 2023-09-06T20:14:47,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031347427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031347428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031347428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031347429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031347531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031347531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031347532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031347532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031347734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031347734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031347734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031347736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:47,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/1080f9fc074d4780bd7742086675d2d3 2023-09-06T20:14:47,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/fd0efe15fd70461cade7d79cca7b9301 is 50, key is test_row_0/B:col10/1694031287279/Put/seqid=0 2023-09-06T20:14:47,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/fd0efe15fd70461cade7d79cca7b9301 2023-09-06T20:14:47,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/1dc8c2bc27c5492ab3bfae4d8e402017 is 50, key is test_row_0/C:col10/1694031287279/Put/seqid=0 2023-09-06T20:14:48,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031348037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031348038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031348039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031348039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031348193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,196 DEBUG [Thread-1443 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4142 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., hostname=jenkins-hbase3.apache.org,33275,1694031217268, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:130) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-06T20:14:48,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/1dc8c2bc27c5492ab3bfae4d8e402017 2023-09-06T20:14:48,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/1080f9fc074d4780bd7742086675d2d3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/1080f9fc074d4780bd7742086675d2d3 2023-09-06T20:14:48,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/1080f9fc074d4780bd7742086675d2d3, entries=150, sequenceid=268, filesize=12.0 K 2023-09-06T20:14:48,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/fd0efe15fd70461cade7d79cca7b9301 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fd0efe15fd70461cade7d79cca7b9301 2023-09-06T20:14:48,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fd0efe15fd70461cade7d79cca7b9301, entries=150, sequenceid=268, filesize=12.0 K 2023-09-06T20:14:48,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/1dc8c2bc27c5492ab3bfae4d8e402017 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/1dc8c2bc27c5492ab3bfae4d8e402017 2023-09-06T20:14:48,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/1dc8c2bc27c5492ab3bfae4d8e402017, entries=150, sequenceid=268, filesize=12.0 K 2023-09-06T20:14:48,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for e66eccf326cb10336c74bdb6873cf7b9 in 869ms, sequenceid=268, compaction requested=true 2023-09-06T20:14:48,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:48,252 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:48,252 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:48,253 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37098 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:48,253 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37098 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:48,253 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/A is initiating minor compaction (all files) 2023-09-06T20:14:48,253 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/B is initiating minor compaction (all files) 2023-09-06T20:14:48,253 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/A in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:48,253 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/B in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:48,253 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/036d435a627c48768b7e5e99466ff638, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d124867072cd4a2b81ca098570ddaee4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/1080f9fc074d4780bd7742086675d2d3] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=36.2 K 2023-09-06T20:14:48,253 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/58252bd410ca413da28d725b7e13b98d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/36cafd16d7804eecbdb33e54aea7fc26, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fd0efe15fd70461cade7d79cca7b9301] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=36.2 K 2023-09-06T20:14:48,254 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 036d435a627c48768b7e5e99466ff638, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1694031285118 2023-09-06T20:14:48,254 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 58252bd410ca413da28d725b7e13b98d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1694031285118 2023-09-06T20:14:48,255 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting d124867072cd4a2b81ca098570ddaee4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1694031285135 2023-09-06T20:14:48,255 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 36cafd16d7804eecbdb33e54aea7fc26, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1694031285135 2023-09-06T20:14:48,255 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1080f9fc074d4780bd7742086675d2d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1694031287277 2023-09-06T20:14:48,256 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting fd0efe15fd70461cade7d79cca7b9301, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1694031287277 2023-09-06T20:14:48,263 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#A#compaction#278 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:48,263 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:48,264 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/67dba75690674d9b9f9c56e3beb43ab0 is 50, key is test_row_0/A:col10/1694031287279/Put/seqid=0 2023-09-06T20:14:48,264 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/95d56aaede1a42d99e23e30b5421b9ea is 50, key is test_row_0/B:col10/1694031287279/Put/seqid=0 2023-09-06T20:14:48,274 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/95d56aaede1a42d99e23e30b5421b9ea as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/95d56aaede1a42d99e23e30b5421b9ea 2023-09-06T20:14:48,279 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/B of e66eccf326cb10336c74bdb6873cf7b9 into 95d56aaede1a42d99e23e30b5421b9ea(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:48,279 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:48,279 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/B, priority=13, startTime=1694031288252; duration=0sec 2023-09-06T20:14:48,279 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:48,279 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:B 2023-09-06T20:14:48,279 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:48,281 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37098 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:48,281 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/C is initiating minor compaction (all files) 2023-09-06T20:14:48,281 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/C in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:48,281 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/25355f9d38de42fc8d95dc63e2af23e8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/ae46ee0300dd41f6b35f370b3c105699, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/1dc8c2bc27c5492ab3bfae4d8e402017] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=36.2 K 2023-09-06T20:14:48,281 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 25355f9d38de42fc8d95dc63e2af23e8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1694031285118 2023-09-06T20:14:48,281 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting ae46ee0300dd41f6b35f370b3c105699, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1694031285135 2023-09-06T20:14:48,282 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 1dc8c2bc27c5492ab3bfae4d8e402017, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1694031287277 2023-09-06T20:14:48,289 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#C#compaction#279 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:48,290 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/5e7dc257a48543adb350660212e216ab is 50, key is test_row_0/C:col10/1694031287279/Put/seqid=0 2023-09-06T20:14:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:48,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2023-09-06T20:14:48,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:48,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:48,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:48,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:48,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:48,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:48,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/65921ef378e1423d84b1ba7b0be3bee5 is 50, key is test_row_0/A:col10/1694031287427/Put/seqid=0 2023-09-06T20:14:48,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031348579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031348580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031348581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031348581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,674 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/67dba75690674d9b9f9c56e3beb43ab0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/67dba75690674d9b9f9c56e3beb43ab0 2023-09-06T20:14:48,680 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/A of e66eccf326cb10336c74bdb6873cf7b9 into 67dba75690674d9b9f9c56e3beb43ab0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:48,680 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:48,680 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/A, priority=13, startTime=1694031288252; duration=0sec 2023-09-06T20:14:48,680 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:48,680 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:A 2023-09-06T20:14:48,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031348684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031348684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031348684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031348684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,722 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/5e7dc257a48543adb350660212e216ab as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/5e7dc257a48543adb350660212e216ab 2023-09-06T20:14:48,727 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/C of e66eccf326cb10336c74bdb6873cf7b9 into 5e7dc257a48543adb350660212e216ab(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:48,727 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:48,727 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/C, priority=13, startTime=1694031288252; duration=0sec 2023-09-06T20:14:48,727 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:48,727 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:C 2023-09-06T20:14:48,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031348896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031348896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031348896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031348896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:48,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/65921ef378e1423d84b1ba7b0be3bee5 2023-09-06T20:14:48,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/1d6387d3f6da49c5b69739c12a9f0ffd is 50, key is test_row_0/B:col10/1694031287427/Put/seqid=0 2023-09-06T20:14:49,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-06T20:14:49,174 INFO [Thread-1447 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 85 completed 2023-09-06T20:14:49,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2023-09-06T20:14:49,178 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-06T20:14:49,179 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:49,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:49,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031349200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:49,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031349200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:49,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031349201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:49,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031349201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-06T20:14:49,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2023-09-06T20:14:49,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:49,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:49,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:49,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(53): pid=88 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:49,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:49,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:49,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/1d6387d3f6da49c5b69739c12a9f0ffd 2023-09-06T20:14:49,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/11272f9010274fbeb34c54ce9d86362d is 50, key is test_row_0/C:col10/1694031287427/Put/seqid=0 2023-09-06T20:14:49,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/11272f9010274fbeb34c54ce9d86362d 2023-09-06T20:14:49,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/65921ef378e1423d84b1ba7b0be3bee5 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/65921ef378e1423d84b1ba7b0be3bee5 2023-09-06T20:14:49,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/65921ef378e1423d84b1ba7b0be3bee5, entries=150, sequenceid=291, filesize=12.0 K 2023-09-06T20:14:49,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/1d6387d3f6da49c5b69739c12a9f0ffd as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/1d6387d3f6da49c5b69739c12a9f0ffd 2023-09-06T20:14:49,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/1d6387d3f6da49c5b69739c12a9f0ffd, entries=150, sequenceid=291, filesize=12.0 K 2023-09-06T20:14:49,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/11272f9010274fbeb34c54ce9d86362d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/11272f9010274fbeb34c54ce9d86362d 2023-09-06T20:14:49,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/11272f9010274fbeb34c54ce9d86362d, entries=150, sequenceid=291, filesize=12.0 K 2023-09-06T20:14:49,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for e66eccf326cb10336c74bdb6873cf7b9 in 855ms, sequenceid=291, compaction requested=false 2023-09-06T20:14:49,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-06T20:14:49,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2023-09-06T20:14:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:49,487 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-06T20:14:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:49,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/be3a4519f1284cad90eb9e41b0310c17 is 50, key is test_row_0/A:col10/1694031288580/Put/seqid=0 2023-09-06T20:14:49,495 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/be3a4519f1284cad90eb9e41b0310c17 2023-09-06T20:14:49,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/8bd012a329d945cd9ad03092000de2d4 is 50, key is test_row_0/B:col10/1694031288580/Put/seqid=0 2023-09-06T20:14:49,512 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/8bd012a329d945cd9ad03092000de2d4 2023-09-06T20:14:49,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/02469cad4b584f7d82d98fea94274560 is 50, key is test_row_0/C:col10/1694031288580/Put/seqid=0 2023-09-06T20:14:49,533 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/02469cad4b584f7d82d98fea94274560 2023-09-06T20:14:49,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/be3a4519f1284cad90eb9e41b0310c17 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/be3a4519f1284cad90eb9e41b0310c17 2023-09-06T20:14:49,541 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/be3a4519f1284cad90eb9e41b0310c17, entries=150, sequenceid=308, filesize=12.0 K 2023-09-06T20:14:49,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/8bd012a329d945cd9ad03092000de2d4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8bd012a329d945cd9ad03092000de2d4 2023-09-06T20:14:49,547 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8bd012a329d945cd9ad03092000de2d4, entries=150, sequenceid=308, filesize=12.0 K 2023-09-06T20:14:49,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/02469cad4b584f7d82d98fea94274560 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/02469cad4b584f7d82d98fea94274560 2023-09-06T20:14:49,552 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/02469cad4b584f7d82d98fea94274560, entries=150, sequenceid=308, filesize=12.0 K 2023-09-06T20:14:49,553 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for e66eccf326cb10336c74bdb6873cf7b9 in 66ms, sequenceid=308, compaction requested=true 2023-09-06T20:14:49,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:49,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:49,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=88 2023-09-06T20:14:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=88 2023-09-06T20:14:49,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=88, resume processing ppid=87 2023-09-06T20:14:49,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=88, ppid=87, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 375 msec 2023-09-06T20:14:49,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=87, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 380 msec 2023-09-06T20:14:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:49,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:14:49,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:49,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:49,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:49,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:49,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:49,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:49,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/fd3ff8a5c84a4de9b3a9e18f356f48a0 is 50, key is test_row_0/A:col10/1694031289708/Put/seqid=0 2023-09-06T20:14:49,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/fd3ff8a5c84a4de9b3a9e18f356f48a0 2023-09-06T20:14:49,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/8c6f3ac5b04e4b538894bc9ef82311c9 is 50, key is test_row_0/B:col10/1694031289708/Put/seqid=0 2023-09-06T20:14:49,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031349733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:49,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031349760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:49,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031349760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:49,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031349760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-06T20:14:49,781 INFO [Thread-1447 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 87 completed 2023-09-06T20:14:49,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2023-09-06T20:14:49,785 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:49,786 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:49,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-06T20:14:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031349861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:49,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031349865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031349865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031349865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-06T20:14:49,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2023-09-06T20:14:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:49,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(53): pid=90 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:50,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031350065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:50,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031350067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:50,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031350068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:50,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031350069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-06T20:14:50,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2023-09-06T20:14:50,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:50,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:50,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:50,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(53): pid=90 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:50,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:50,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/8c6f3ac5b04e4b538894bc9ef82311c9 2023-09-06T20:14:50,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/147fb2985937456a8c55d017f212196e is 50, key is test_row_0/C:col10/1694031289708/Put/seqid=0 2023-09-06T20:14:50,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2023-09-06T20:14:50,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:50,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:50,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:50,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(53): pid=90 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:50,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:50,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031350368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:50,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:50,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031350371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:50,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:50,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031350373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:50,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:50,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031350373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:50,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-06T20:14:50,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2023-09-06T20:14:50,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:50,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:50,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:50,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(53): pid=90 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:50,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:50,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:50,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/147fb2985937456a8c55d017f212196e 2023-09-06T20:14:50,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2023-09-06T20:14:50,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:50,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:50,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:50,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(53): pid=90 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:50,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:50,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:50,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/fd3ff8a5c84a4de9b3a9e18f356f48a0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/fd3ff8a5c84a4de9b3a9e18f356f48a0 2023-09-06T20:14:50,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/fd3ff8a5c84a4de9b3a9e18f356f48a0, entries=150, sequenceid=319, filesize=12.0 K 2023-09-06T20:14:50,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/8c6f3ac5b04e4b538894bc9ef82311c9 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8c6f3ac5b04e4b538894bc9ef82311c9 2023-09-06T20:14:50,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8c6f3ac5b04e4b538894bc9ef82311c9, entries=150, sequenceid=319, filesize=12.0 K 2023-09-06T20:14:50,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/147fb2985937456a8c55d017f212196e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/147fb2985937456a8c55d017f212196e 2023-09-06T20:14:50,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/147fb2985937456a8c55d017f212196e, entries=150, sequenceid=319, filesize=12.0 K 2023-09-06T20:14:50,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for e66eccf326cb10336c74bdb6873cf7b9 in 859ms, sequenceid=319, compaction requested=true 2023-09-06T20:14:50,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:50,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:50,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:50,570 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:50,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:50,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:50,570 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:50,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:50,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:50,571 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:50,571 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:50,571 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/B is initiating minor compaction (all files) 2023-09-06T20:14:50,571 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/A is initiating minor compaction (all files) 2023-09-06T20:14:50,571 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/B in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:50,571 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/A in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:50,571 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/95d56aaede1a42d99e23e30b5421b9ea, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/1d6387d3f6da49c5b69739c12a9f0ffd, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8bd012a329d945cd9ad03092000de2d4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8c6f3ac5b04e4b538894bc9ef82311c9] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=48.6 K 2023-09-06T20:14:50,571 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/67dba75690674d9b9f9c56e3beb43ab0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/65921ef378e1423d84b1ba7b0be3bee5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/be3a4519f1284cad90eb9e41b0310c17, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/fd3ff8a5c84a4de9b3a9e18f356f48a0] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=48.6 K 2023-09-06T20:14:50,572 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 67dba75690674d9b9f9c56e3beb43ab0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1694031287277 2023-09-06T20:14:50,572 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 95d56aaede1a42d99e23e30b5421b9ea, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1694031287277 2023-09-06T20:14:50,572 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 65921ef378e1423d84b1ba7b0be3bee5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1694031287427 2023-09-06T20:14:50,572 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 1d6387d3f6da49c5b69739c12a9f0ffd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1694031287427 2023-09-06T20:14:50,573 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting be3a4519f1284cad90eb9e41b0310c17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1694031288578 2023-09-06T20:14:50,573 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 8bd012a329d945cd9ad03092000de2d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1694031288578 2023-09-06T20:14:50,573 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting fd3ff8a5c84a4de9b3a9e18f356f48a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1694031289708 2023-09-06T20:14:50,573 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 8c6f3ac5b04e4b538894bc9ef82311c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1694031289708 2023-09-06T20:14:50,588 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#A#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:50,588 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/7ff253ef60994aaeaf02e16450ef6e69 is 50, key is test_row_0/A:col10/1694031289708/Put/seqid=0 2023-09-06T20:14:50,589 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#B#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:50,589 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/5375f09e8e244b0f92de89fd4eea86d7 is 50, key is test_row_0/B:col10/1694031289708/Put/seqid=0 2023-09-06T20:14:50,607 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/5375f09e8e244b0f92de89fd4eea86d7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/5375f09e8e244b0f92de89fd4eea86d7 2023-09-06T20:14:50,612 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/B of e66eccf326cb10336c74bdb6873cf7b9 into 5375f09e8e244b0f92de89fd4eea86d7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:50,612 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:50,612 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/B, priority=12, startTime=1694031290570; duration=0sec 2023-09-06T20:14:50,612 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:50,612 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:B 2023-09-06T20:14:50,612 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:14:50,614 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:14:50,614 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/C is initiating minor compaction (all files) 2023-09-06T20:14:50,614 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/C in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:50,614 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/5e7dc257a48543adb350660212e216ab, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/11272f9010274fbeb34c54ce9d86362d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/02469cad4b584f7d82d98fea94274560, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/147fb2985937456a8c55d017f212196e] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=48.6 K 2023-09-06T20:14:50,614 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 5e7dc257a48543adb350660212e216ab, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1694031287277 2023-09-06T20:14:50,614 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 11272f9010274fbeb34c54ce9d86362d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1694031287427 2023-09-06T20:14:50,615 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 02469cad4b584f7d82d98fea94274560, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1694031288578 2023-09-06T20:14:50,615 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 147fb2985937456a8c55d017f212196e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1694031289708 2023-09-06T20:14:50,623 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#C#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:50,624 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/52628cb37e9d401cb4481f1e4875da6e is 50, key is test_row_0/C:col10/1694031289708/Put/seqid=0 2023-09-06T20:14:50,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2023-09-06T20:14:50,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:50,705 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-06T20:14:50,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:50,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:50,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:50,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:50,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:50,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:50,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/99948f37fe084c28baabb4f8655cea87 is 50, key is test_row_0/A:col10/1694031289737/Put/seqid=0 2023-09-06T20:14:50,719 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/99948f37fe084c28baabb4f8655cea87 2023-09-06T20:14:50,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/9719893d2b424528abb5fbf03936f7d7 is 50, key is test_row_0/B:col10/1694031289737/Put/seqid=0 2023-09-06T20:14:50,739 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/9719893d2b424528abb5fbf03936f7d7 2023-09-06T20:14:50,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/a123a9e1bff143d3b6ae7946d36f963b is 50, key is test_row_0/C:col10/1694031289737/Put/seqid=0 2023-09-06T20:14:50,752 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/a123a9e1bff143d3b6ae7946d36f963b 2023-09-06T20:14:50,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/99948f37fe084c28baabb4f8655cea87 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/99948f37fe084c28baabb4f8655cea87 2023-09-06T20:14:50,760 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/99948f37fe084c28baabb4f8655cea87, entries=150, sequenceid=346, filesize=12.0 K 2023-09-06T20:14:50,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/9719893d2b424528abb5fbf03936f7d7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/9719893d2b424528abb5fbf03936f7d7 2023-09-06T20:14:50,765 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/9719893d2b424528abb5fbf03936f7d7, entries=150, sequenceid=346, filesize=12.0 K 2023-09-06T20:14:50,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/a123a9e1bff143d3b6ae7946d36f963b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/a123a9e1bff143d3b6ae7946d36f963b 2023-09-06T20:14:50,775 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/a123a9e1bff143d3b6ae7946d36f963b, entries=150, sequenceid=346, filesize=12.0 K 2023-09-06T20:14:50,775 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for e66eccf326cb10336c74bdb6873cf7b9 in 70ms, sequenceid=346, compaction requested=false 2023-09-06T20:14:50,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=90 2023-09-06T20:14:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=90 2023-09-06T20:14:50,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=90, resume processing ppid=89 2023-09-06T20:14:50,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=90, ppid=89, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 991 msec 2023-09-06T20:14:50,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 995 msec 2023-09-06T20:14:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:50,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:14:50,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:50,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:50,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:50,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:50,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:50,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:50,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/479ccee0aef244bfbcd8207ec85e86b7 is 50, key is test_row_0/A:col10/1694031290882/Put/seqid=0 2023-09-06T20:14:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-06T20:14:50,890 INFO [Thread-1447 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 89 completed 2023-09-06T20:14:50,891 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2023-09-06T20:14:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-06T20:14:50,894 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:50,894 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:50,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:50,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031350948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:50,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031350949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:50,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031350949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:50,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031350950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-06T20:14:51,001 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/7ff253ef60994aaeaf02e16450ef6e69 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/7ff253ef60994aaeaf02e16450ef6e69 2023-09-06T20:14:51,005 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/A of e66eccf326cb10336c74bdb6873cf7b9 into 7ff253ef60994aaeaf02e16450ef6e69(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:51,005 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:51,005 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/A, priority=12, startTime=1694031290570; duration=0sec 2023-09-06T20:14:51,005 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:51,005 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:A 2023-09-06T20:14:51,033 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/52628cb37e9d401cb4481f1e4875da6e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/52628cb37e9d401cb4481f1e4875da6e 2023-09-06T20:14:51,037 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/C of e66eccf326cb10336c74bdb6873cf7b9 into 52628cb37e9d401cb4481f1e4875da6e(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:51,037 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:51,037 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/C, priority=12, startTime=1694031290570; duration=0sec 2023-09-06T20:14:51,037 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:51,038 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:C 2023-09-06T20:14:51,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2023-09-06T20:14:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(53): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031351051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:51,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031351054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:51,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031351054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:51,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031351054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-06T20:14:51,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2023-09-06T20:14:51,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:51,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(53): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031351253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:51,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031351256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:51,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031351256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:51,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031351257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:51,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/479ccee0aef244bfbcd8207ec85e86b7 2023-09-06T20:14:51,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/424b9c137abf40a7b8439b7acf037f2c is 50, key is test_row_0/B:col10/1694031290882/Put/seqid=0 2023-09-06T20:14:51,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2023-09-06T20:14:51,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:51,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(53): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-06T20:14:51,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2023-09-06T20:14:51,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:51,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(53): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031351557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:51,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031351560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:51,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031351560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:51,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031351560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:51,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2023-09-06T20:14:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(53): pid=92 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:51,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/424b9c137abf40a7b8439b7acf037f2c 2023-09-06T20:14:51,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/d57766b1948a416d9588fe461f80d5c8 is 50, key is test_row_0/C:col10/1694031290882/Put/seqid=0 2023-09-06T20:14:51,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/d57766b1948a416d9588fe461f80d5c8 2023-09-06T20:14:51,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/479ccee0aef244bfbcd8207ec85e86b7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/479ccee0aef244bfbcd8207ec85e86b7 2023-09-06T20:14:51,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/479ccee0aef244bfbcd8207ec85e86b7, entries=150, sequenceid=358, filesize=12.0 K 2023-09-06T20:14:51,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/424b9c137abf40a7b8439b7acf037f2c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/424b9c137abf40a7b8439b7acf037f2c 2023-09-06T20:14:51,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/424b9c137abf40a7b8439b7acf037f2c, entries=150, sequenceid=358, filesize=12.0 K 2023-09-06T20:14:51,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/d57766b1948a416d9588fe461f80d5c8 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/d57766b1948a416d9588fe461f80d5c8 2023-09-06T20:14:51,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/d57766b1948a416d9588fe461f80d5c8, entries=150, sequenceid=358, filesize=12.0 K 2023-09-06T20:14:51,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for e66eccf326cb10336c74bdb6873cf7b9 in 881ms, sequenceid=358, compaction requested=true 2023-09-06T20:14:51,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:51,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:51,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:51,765 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:51,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:51,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:51,765 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:51,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:51,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:51,766 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37636 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:51,766 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/A is initiating minor compaction (all files) 2023-09-06T20:14:51,766 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37636 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:51,766 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/A in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,766 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/B is initiating minor compaction (all files) 2023-09-06T20:14:51,766 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/B in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,766 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/7ff253ef60994aaeaf02e16450ef6e69, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/99948f37fe084c28baabb4f8655cea87, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/479ccee0aef244bfbcd8207ec85e86b7] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=36.8 K 2023-09-06T20:14:51,766 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/5375f09e8e244b0f92de89fd4eea86d7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/9719893d2b424528abb5fbf03936f7d7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/424b9c137abf40a7b8439b7acf037f2c] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=36.8 K 2023-09-06T20:14:51,767 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7ff253ef60994aaeaf02e16450ef6e69, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1694031289708 2023-09-06T20:14:51,767 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 5375f09e8e244b0f92de89fd4eea86d7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1694031289708 2023-09-06T20:14:51,767 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 99948f37fe084c28baabb4f8655cea87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1694031289731 2023-09-06T20:14:51,768 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 9719893d2b424528abb5fbf03936f7d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1694031289731 2023-09-06T20:14:51,768 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 479ccee0aef244bfbcd8207ec85e86b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1694031290877 2023-09-06T20:14:51,768 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 424b9c137abf40a7b8439b7acf037f2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1694031290877 2023-09-06T20:14:51,775 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#A#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:51,776 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/daf5a5c0920844c6853f0fcba849023b is 50, key is test_row_0/A:col10/1694031290882/Put/seqid=0 2023-09-06T20:14:51,783 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#B#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:51,783 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/fc8f5b69d56d452aa606141fbcf1031a is 50, key is test_row_0/B:col10/1694031290882/Put/seqid=0 2023-09-06T20:14:51,791 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/daf5a5c0920844c6853f0fcba849023b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/daf5a5c0920844c6853f0fcba849023b 2023-09-06T20:14:51,797 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/A of e66eccf326cb10336c74bdb6873cf7b9 into daf5a5c0920844c6853f0fcba849023b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:51,797 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:51,797 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/A, priority=13, startTime=1694031291765; duration=0sec 2023-09-06T20:14:51,797 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:51,798 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:A 2023-09-06T20:14:51,798 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:51,799 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37636 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:51,799 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/C is initiating minor compaction (all files) 2023-09-06T20:14:51,799 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/C in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,799 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/52628cb37e9d401cb4481f1e4875da6e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/a123a9e1bff143d3b6ae7946d36f963b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/d57766b1948a416d9588fe461f80d5c8] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=36.8 K 2023-09-06T20:14:51,799 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 52628cb37e9d401cb4481f1e4875da6e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1694031289708 2023-09-06T20:14:51,800 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting a123a9e1bff143d3b6ae7946d36f963b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1694031289731 2023-09-06T20:14:51,800 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting d57766b1948a416d9588fe461f80d5c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1694031290877 2023-09-06T20:14:51,801 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/fc8f5b69d56d452aa606141fbcf1031a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fc8f5b69d56d452aa606141fbcf1031a 2023-09-06T20:14:51,807 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/B of e66eccf326cb10336c74bdb6873cf7b9 into fc8f5b69d56d452aa606141fbcf1031a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:51,807 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:51,807 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/B, priority=13, startTime=1694031291765; duration=0sec 2023-09-06T20:14:51,807 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:51,807 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:B 2023-09-06T20:14:51,808 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#C#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:51,808 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/acd78159a8fe441f9c01ad9482f4de3e is 50, key is test_row_0/C:col10/1694031290882/Put/seqid=0 2023-09-06T20:14:51,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2023-09-06T20:14:51,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,813 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-06T20:14:51,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:51,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:51,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:51,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:51,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:51,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:51,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/b8caecdc64c44b09bc119410ea722b18 is 50, key is test_row_0/A:col10/1694031290948/Put/seqid=0 2023-09-06T20:14:51,822 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/b8caecdc64c44b09bc119410ea722b18 2023-09-06T20:14:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/770623d98f9443e5a275c567e9656159 is 50, key is test_row_0/B:col10/1694031290948/Put/seqid=0 2023-09-06T20:14:51,874 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/770623d98f9443e5a275c567e9656159 2023-09-06T20:14:51,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/c49f1e9d26e644f6a1d0bc60db9530e0 is 50, key is test_row_0/C:col10/1694031290948/Put/seqid=0 2023-09-06T20:14:51,902 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/c49f1e9d26e644f6a1d0bc60db9530e0 2023-09-06T20:14:51,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/b8caecdc64c44b09bc119410ea722b18 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b8caecdc64c44b09bc119410ea722b18 2023-09-06T20:14:51,910 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b8caecdc64c44b09bc119410ea722b18, entries=150, sequenceid=387, filesize=12.0 K 2023-09-06T20:14:51,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/770623d98f9443e5a275c567e9656159 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/770623d98f9443e5a275c567e9656159 2023-09-06T20:14:51,923 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/770623d98f9443e5a275c567e9656159, entries=150, sequenceid=387, filesize=12.0 K 2023-09-06T20:14:51,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/c49f1e9d26e644f6a1d0bc60db9530e0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/c49f1e9d26e644f6a1d0bc60db9530e0 2023-09-06T20:14:51,927 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/c49f1e9d26e644f6a1d0bc60db9530e0, entries=150, sequenceid=387, filesize=12.0 K 2023-09-06T20:14:51,928 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for e66eccf326cb10336c74bdb6873cf7b9 in 115ms, sequenceid=387, compaction requested=false 2023-09-06T20:14:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=92 2023-09-06T20:14:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=92 2023-09-06T20:14:51,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=92, resume processing ppid=91 2023-09-06T20:14:51,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=92, ppid=91, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0370 sec 2023-09-06T20:14:51,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.0420 sec 2023-09-06T20:14:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-06T20:14:51,997 INFO [Thread-1447 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 91 completed 2023-09-06T20:14:51,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2023-09-06T20:14:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-06T20:14:52,002 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:52,002 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:52,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:52,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:14:52,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:52,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:52,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:52,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:52,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:52,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:52,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/baa207d5524e4f1a95d4b13ec4afaea3 is 50, key is test_row_0/A:col10/1694031292077/Put/seqid=0 2023-09-06T20:14:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-06T20:14:52,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031352109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031352109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031352109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031352111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2023-09-06T20:14:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:52,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(53): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:52,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031352214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031352214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031352214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031352214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,221 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/acd78159a8fe441f9c01ad9482f4de3e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/acd78159a8fe441f9c01ad9482f4de3e 2023-09-06T20:14:52,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48158 deadline: 1694031352222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,225 DEBUG [Thread-1443 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8171 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., hostname=jenkins-hbase3.apache.org,33275,1694031217268, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:130) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-06T20:14:52,227 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/C of e66eccf326cb10336c74bdb6873cf7b9 into acd78159a8fe441f9c01ad9482f4de3e(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:52,227 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:52,227 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/C, priority=13, startTime=1694031291765; duration=0sec 2023-09-06T20:14:52,227 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:52,227 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:C 2023-09-06T20:14:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-06T20:14:52,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2023-09-06T20:14:52,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:52,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:52,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:52,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(53): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:52,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:52,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031352416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031352418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031352418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031352426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2023-09-06T20:14:52,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:52,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:52,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:52,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(53): pid=94 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:52,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:52,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/baa207d5524e4f1a95d4b13ec4afaea3 2023-09-06T20:14:52,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/a8280ecb5156411889312f2c67a78669 is 50, key is test_row_0/B:col10/1694031292077/Put/seqid=0 2023-09-06T20:14:52,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/a8280ecb5156411889312f2c67a78669 2023-09-06T20:14:52,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/acffe2fe15b0474b89983af52c6a6d5c is 50, key is test_row_0/C:col10/1694031292077/Put/seqid=0 2023-09-06T20:14:52,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/acffe2fe15b0474b89983af52c6a6d5c 2023-09-06T20:14:52,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/baa207d5524e4f1a95d4b13ec4afaea3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/baa207d5524e4f1a95d4b13ec4afaea3 2023-09-06T20:14:52,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/baa207d5524e4f1a95d4b13ec4afaea3, entries=200, sequenceid=400, filesize=14.4 K 2023-09-06T20:14:52,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/a8280ecb5156411889312f2c67a78669 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a8280ecb5156411889312f2c67a78669 2023-09-06T20:14:52,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a8280ecb5156411889312f2c67a78669, entries=150, sequenceid=400, filesize=12.0 K 2023-09-06T20:14:52,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/acffe2fe15b0474b89983af52c6a6d5c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/acffe2fe15b0474b89983af52c6a6d5c 2023-09-06T20:14:52,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/acffe2fe15b0474b89983af52c6a6d5c, entries=150, sequenceid=400, filesize=12.0 K 2023-09-06T20:14:52,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e66eccf326cb10336c74bdb6873cf7b9 in 490ms, sequenceid=400, compaction requested=true 2023-09-06T20:14:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:52,570 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:52,570 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:52,571 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40178 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:52,571 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/A is initiating minor compaction (all files) 2023-09-06T20:14:52,571 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37738 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:52,571 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/B is initiating minor compaction (all files) 2023-09-06T20:14:52,571 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/A in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:52,571 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/B in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:52,571 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/daf5a5c0920844c6853f0fcba849023b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b8caecdc64c44b09bc119410ea722b18, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/baa207d5524e4f1a95d4b13ec4afaea3] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=39.2 K 2023-09-06T20:14:52,571 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fc8f5b69d56d452aa606141fbcf1031a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/770623d98f9443e5a275c567e9656159, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a8280ecb5156411889312f2c67a78669] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=36.9 K 2023-09-06T20:14:52,572 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting daf5a5c0920844c6853f0fcba849023b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1694031290877 2023-09-06T20:14:52,572 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting fc8f5b69d56d452aa606141fbcf1031a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1694031290877 2023-09-06T20:14:52,572 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting b8caecdc64c44b09bc119410ea722b18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1694031290948 2023-09-06T20:14:52,572 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 770623d98f9443e5a275c567e9656159, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1694031290948 2023-09-06T20:14:52,573 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting baa207d5524e4f1a95d4b13ec4afaea3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1694031292076 2023-09-06T20:14:52,573 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting a8280ecb5156411889312f2c67a78669, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1694031292077 2023-09-06T20:14:52,579 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#B#compaction#307 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:52,580 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/d42fbefce0af4f7ba21be86f5590ff2b is 50, key is test_row_0/B:col10/1694031292077/Put/seqid=0 2023-09-06T20:14:52,582 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#A#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:52,582 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/482cb447a12f47ea9ca840534ea592a7 is 50, key is test_row_0/A:col10/1694031292077/Put/seqid=0 2023-09-06T20:14:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-06T20:14:52,611 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/482cb447a12f47ea9ca840534ea592a7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/482cb447a12f47ea9ca840534ea592a7 2023-09-06T20:14:52,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2023-09-06T20:14:52,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:52,614 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-06T20:14:52,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:52,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:52,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:52,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:52,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:52,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:52,616 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/A of e66eccf326cb10336c74bdb6873cf7b9 into 482cb447a12f47ea9ca840534ea592a7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:52,616 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:52,616 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/A, priority=13, startTime=1694031292570; duration=0sec 2023-09-06T20:14:52,616 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:52,616 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:A 2023-09-06T20:14:52,616 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:52,619 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37738 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:52,619 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/C is initiating minor compaction (all files) 2023-09-06T20:14:52,619 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/C in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:52,619 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/acd78159a8fe441f9c01ad9482f4de3e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/c49f1e9d26e644f6a1d0bc60db9530e0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/acffe2fe15b0474b89983af52c6a6d5c] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=36.9 K 2023-09-06T20:14:52,619 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting acd78159a8fe441f9c01ad9482f4de3e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1694031290877 2023-09-06T20:14:52,620 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting c49f1e9d26e644f6a1d0bc60db9530e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1694031290948 2023-09-06T20:14:52,620 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting acffe2fe15b0474b89983af52c6a6d5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1694031292077 2023-09-06T20:14:52,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/6b68ac38e1fa4edb9c102cc6774616b1 is 50, key is test_row_0/A:col10/1694031292102/Put/seqid=0 2023-09-06T20:14:52,627 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#C#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:52,627 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/3978b65a6a2445d69e1e061bd70da9ce is 50, key is test_row_0/C:col10/1694031292077/Put/seqid=0 2023-09-06T20:14:52,654 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/6b68ac38e1fa4edb9c102cc6774616b1 2023-09-06T20:14:52,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/a1675524ec5a46f2ae1dbd15da188c48 is 50, key is test_row_0/B:col10/1694031292102/Put/seqid=0 2023-09-06T20:14:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:52,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:52,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031352731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031352732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031352733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031352734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031352836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031352837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:52,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031352838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:53,008 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/d42fbefce0af4f7ba21be86f5590ff2b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d42fbefce0af4f7ba21be86f5590ff2b 2023-09-06T20:14:53,012 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/B of e66eccf326cb10336c74bdb6873cf7b9 into d42fbefce0af4f7ba21be86f5590ff2b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:53,012 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:53,012 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/B, priority=13, startTime=1694031292570; duration=0sec 2023-09-06T20:14:53,012 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:53,012 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:B 2023-09-06T20:14:53,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031353038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:53,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031353040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:53,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031353041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:53,062 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/3978b65a6a2445d69e1e061bd70da9ce as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/3978b65a6a2445d69e1e061bd70da9ce 2023-09-06T20:14:53,067 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/C of e66eccf326cb10336c74bdb6873cf7b9 into 3978b65a6a2445d69e1e061bd70da9ce(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:53,067 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:53,067 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/C, priority=13, startTime=1694031292570; duration=0sec 2023-09-06T20:14:53,067 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:53,067 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:C 2023-09-06T20:14:53,088 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/a1675524ec5a46f2ae1dbd15da188c48 2023-09-06T20:14:53,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/8617eeb41a494fd9a85f3c5bf09eab4d is 50, key is test_row_0/C:col10/1694031292102/Put/seqid=0 2023-09-06T20:14:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-06T20:14:53,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:53,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031353234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:53,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031353343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:53,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031353344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:53,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031353346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:53,520 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/8617eeb41a494fd9a85f3c5bf09eab4d 2023-09-06T20:14:53,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/6b68ac38e1fa4edb9c102cc6774616b1 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/6b68ac38e1fa4edb9c102cc6774616b1 2023-09-06T20:14:53,533 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/6b68ac38e1fa4edb9c102cc6774616b1, entries=150, sequenceid=424, filesize=12.0 K 2023-09-06T20:14:53,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/a1675524ec5a46f2ae1dbd15da188c48 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a1675524ec5a46f2ae1dbd15da188c48 2023-09-06T20:14:53,538 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a1675524ec5a46f2ae1dbd15da188c48, entries=150, sequenceid=424, filesize=12.0 K 2023-09-06T20:14:53,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/8617eeb41a494fd9a85f3c5bf09eab4d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/8617eeb41a494fd9a85f3c5bf09eab4d 2023-09-06T20:14:53,542 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/8617eeb41a494fd9a85f3c5bf09eab4d, entries=150, sequenceid=424, filesize=12.0 K 2023-09-06T20:14:53,543 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for e66eccf326cb10336c74bdb6873cf7b9 in 929ms, sequenceid=424, compaction requested=false 2023-09-06T20:14:53,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:53,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:53,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=94 2023-09-06T20:14:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=94 2023-09-06T20:14:53,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=94, resume processing ppid=93 2023-09-06T20:14:53,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=94, ppid=93, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5420 sec 2023-09-06T20:14:53,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=93, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.5470 sec 2023-09-06T20:14:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:53,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-06T20:14:53,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:53,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:53,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:53,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:53,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:53,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:53,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/b825572bca824048acbb094777173a27 is 50, key is test_row_0/A:col10/1694031293847/Put/seqid=0 2023-09-06T20:14:53,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031353873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:53,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031353874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:53,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031353875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:53,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031353977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:53,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031353977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:53,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031353978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-06T20:14:54,107 INFO [Thread-1447 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 93 completed 2023-09-06T20:14:54,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2023-09-06T20:14:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-06T20:14:54,111 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:54,111 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:54,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:54,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031354179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:54,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031354180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:54,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031354181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-06T20:14:54,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031354241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:54,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2023-09-06T20:14:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:54,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(53): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:54,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/b825572bca824048acbb094777173a27 2023-09-06T20:14:54,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/ba18ef8dc7644642b7e341a07f86fc06 is 50, key is test_row_0/B:col10/1694031293847/Put/seqid=0 2023-09-06T20:14:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-06T20:14:54,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2023-09-06T20:14:54,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:54,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:54,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:54,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(53): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:54,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:54,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031354482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:54,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031354483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031354486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:54,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2023-09-06T20:14:54,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:54,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:54,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:54,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(53): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:54,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:54,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/ba18ef8dc7644642b7e341a07f86fc06 2023-09-06T20:14:54,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/b145935ef8a446fb8899d48e8f5d34b1 is 50, key is test_row_0/C:col10/1694031293847/Put/seqid=0 2023-09-06T20:14:54,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/b145935ef8a446fb8899d48e8f5d34b1 2023-09-06T20:14:54,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/b825572bca824048acbb094777173a27 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b825572bca824048acbb094777173a27 2023-09-06T20:14:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-06T20:14:54,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b825572bca824048acbb094777173a27, entries=150, sequenceid=442, filesize=12.0 K 2023-09-06T20:14:54,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/ba18ef8dc7644642b7e341a07f86fc06 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/ba18ef8dc7644642b7e341a07f86fc06 2023-09-06T20:14:54,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/ba18ef8dc7644642b7e341a07f86fc06, entries=150, sequenceid=442, filesize=12.0 K 2023-09-06T20:14:54,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2023-09-06T20:14:54,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:54,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/b145935ef8a446fb8899d48e8f5d34b1 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/b145935ef8a446fb8899d48e8f5d34b1 2023-09-06T20:14:54,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:54,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:54,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(53): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:54,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:54,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/b145935ef8a446fb8899d48e8f5d34b1, entries=150, sequenceid=442, filesize=12.0 K 2023-09-06T20:14:54,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for e66eccf326cb10336c74bdb6873cf7b9 in 879ms, sequenceid=442, compaction requested=true 2023-09-06T20:14:54,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:54,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:54,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:54,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:54,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:54,728 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:54,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:54,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:54,728 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:54,729 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37840 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:54,729 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/A is initiating minor compaction (all files) 2023-09-06T20:14:54,729 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/A in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:54,729 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/482cb447a12f47ea9ca840534ea592a7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/6b68ac38e1fa4edb9c102cc6774616b1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b825572bca824048acbb094777173a27] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=37.0 K 2023-09-06T20:14:54,730 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37840 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:54,730 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/B is initiating minor compaction (all files) 2023-09-06T20:14:54,730 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/B in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:54,730 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d42fbefce0af4f7ba21be86f5590ff2b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a1675524ec5a46f2ae1dbd15da188c48, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/ba18ef8dc7644642b7e341a07f86fc06] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=37.0 K 2023-09-06T20:14:54,730 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 482cb447a12f47ea9ca840534ea592a7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1694031292077 2023-09-06T20:14:54,730 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting d42fbefce0af4f7ba21be86f5590ff2b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1694031292077 2023-09-06T20:14:54,730 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6b68ac38e1fa4edb9c102cc6774616b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1694031292102 2023-09-06T20:14:54,731 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting a1675524ec5a46f2ae1dbd15da188c48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1694031292102 2023-09-06T20:14:54,731 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting b825572bca824048acbb094777173a27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1694031292731 2023-09-06T20:14:54,731 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting ba18ef8dc7644642b7e341a07f86fc06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1694031292731 2023-09-06T20:14:54,738 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#A#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:54,739 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/d08c0a63ec904c389acc7d7e938aa9a2 is 50, key is test_row_0/A:col10/1694031293847/Put/seqid=0 2023-09-06T20:14:54,739 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#B#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:54,740 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/3eff0dd6ffd9450dac82c2777d88748f is 50, key is test_row_0/B:col10/1694031293847/Put/seqid=0 2023-09-06T20:14:54,753 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/3eff0dd6ffd9450dac82c2777d88748f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/3eff0dd6ffd9450dac82c2777d88748f 2023-09-06T20:14:54,758 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/B of e66eccf326cb10336c74bdb6873cf7b9 into 3eff0dd6ffd9450dac82c2777d88748f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:54,758 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:54,758 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/B, priority=13, startTime=1694031294728; duration=0sec 2023-09-06T20:14:54,758 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:54,758 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:B 2023-09-06T20:14:54,758 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:54,760 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37840 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:54,760 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/C is initiating minor compaction (all files) 2023-09-06T20:14:54,760 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/C in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:54,760 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/3978b65a6a2445d69e1e061bd70da9ce, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/8617eeb41a494fd9a85f3c5bf09eab4d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/b145935ef8a446fb8899d48e8f5d34b1] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=37.0 K 2023-09-06T20:14:54,760 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 3978b65a6a2445d69e1e061bd70da9ce, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1694031292077 2023-09-06T20:14:54,761 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 8617eeb41a494fd9a85f3c5bf09eab4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1694031292102 2023-09-06T20:14:54,761 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b145935ef8a446fb8899d48e8f5d34b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1694031292731 2023-09-06T20:14:54,768 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#C#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:54,768 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/181d127f87734b8ba39f5bdb202925de is 50, key is test_row_0/C:col10/1694031293847/Put/seqid=0 2023-09-06T20:14:54,779 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/181d127f87734b8ba39f5bdb202925de as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/181d127f87734b8ba39f5bdb202925de 2023-09-06T20:14:54,784 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/C of e66eccf326cb10336c74bdb6873cf7b9 into 181d127f87734b8ba39f5bdb202925de(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:54,784 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:54,784 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/C, priority=13, startTime=1694031294728; duration=0sec 2023-09-06T20:14:54,784 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:54,784 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:C 2023-09-06T20:14:54,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2023-09-06T20:14:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:54,876 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2023-09-06T20:14:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:54,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/5967c7a2fa0b4410b9feb8f23ba4fdc6 is 50, key is test_row_0/A:col10/1694031293873/Put/seqid=0 2023-09-06T20:14:54,885 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/5967c7a2fa0b4410b9feb8f23ba4fdc6 2023-09-06T20:14:54,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/0a794ec6ce3b494cb3bdc79eb0a5030e is 50, key is test_row_0/B:col10/1694031293873/Put/seqid=0 2023-09-06T20:14:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:54,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:55,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031355001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031355002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031355002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031355105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031355106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031355106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,153 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/d08c0a63ec904c389acc7d7e938aa9a2 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d08c0a63ec904c389acc7d7e938aa9a2 2023-09-06T20:14:55,158 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/A of e66eccf326cb10336c74bdb6873cf7b9 into d08c0a63ec904c389acc7d7e938aa9a2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:55,158 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:55,158 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/A, priority=13, startTime=1694031294728; duration=0sec 2023-09-06T20:14:55,158 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:55,158 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:A 2023-09-06T20:14:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-06T20:14:55,302 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/0a794ec6ce3b494cb3bdc79eb0a5030e 2023-09-06T20:14:55,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/896bcc9a6f8e487a85726fba3397b0d2 is 50, key is test_row_0/C:col10/1694031293873/Put/seqid=0 2023-09-06T20:14:55,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031355309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031355309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031355310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,315 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/896bcc9a6f8e487a85726fba3397b0d2 2023-09-06T20:14:55,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/5967c7a2fa0b4410b9feb8f23ba4fdc6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/5967c7a2fa0b4410b9feb8f23ba4fdc6 2023-09-06T20:14:55,322 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/5967c7a2fa0b4410b9feb8f23ba4fdc6, entries=150, sequenceid=465, filesize=12.0 K 2023-09-06T20:14:55,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/0a794ec6ce3b494cb3bdc79eb0a5030e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/0a794ec6ce3b494cb3bdc79eb0a5030e 2023-09-06T20:14:55,328 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/0a794ec6ce3b494cb3bdc79eb0a5030e, entries=150, sequenceid=465, filesize=12.0 K 2023-09-06T20:14:55,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/896bcc9a6f8e487a85726fba3397b0d2 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/896bcc9a6f8e487a85726fba3397b0d2 2023-09-06T20:14:55,332 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/896bcc9a6f8e487a85726fba3397b0d2, entries=150, sequenceid=465, filesize=12.0 K 2023-09-06T20:14:55,333 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3022): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for e66eccf326cb10336c74bdb6873cf7b9 in 456ms, sequenceid=465, compaction requested=false 2023-09-06T20:14:55,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:55,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:55,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=96 2023-09-06T20:14:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=96 2023-09-06T20:14:55,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=96, resume processing ppid=95 2023-09-06T20:14:55,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=96, ppid=95, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2220 sec 2023-09-06T20:14:55,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=95, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.2260 sec 2023-09-06T20:14:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:55,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2023-09-06T20:14:55,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:55,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:55,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:55,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:55,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:55,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:55,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/4c4e0591bf644582b8b3c7e8a73b7eba is 50, key is test_row_0/A:col10/1694031295001/Put/seqid=0 2023-09-06T20:14:55,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/4c4e0591bf644582b8b3c7e8a73b7eba 2023-09-06T20:14:55,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/46c1b64942c542c293dadb3c352b9932 is 50, key is test_row_0/B:col10/1694031295001/Put/seqid=0 2023-09-06T20:14:55,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/46c1b64942c542c293dadb3c352b9932 2023-09-06T20:14:55,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031355634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031355637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031355637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/e278f84a134f407fba2477694fdb753e is 50, key is test_row_0/C:col10/1694031295001/Put/seqid=0 2023-09-06T20:14:55,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/e278f84a134f407fba2477694fdb753e 2023-09-06T20:14:55,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/4c4e0591bf644582b8b3c7e8a73b7eba as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/4c4e0591bf644582b8b3c7e8a73b7eba 2023-09-06T20:14:55,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/4c4e0591bf644582b8b3c7e8a73b7eba, entries=150, sequenceid=482, filesize=12.0 K 2023-09-06T20:14:55,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/46c1b64942c542c293dadb3c352b9932 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/46c1b64942c542c293dadb3c352b9932 2023-09-06T20:14:55,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/46c1b64942c542c293dadb3c352b9932, entries=150, sequenceid=482, filesize=12.0 K 2023-09-06T20:14:55,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/e278f84a134f407fba2477694fdb753e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/e278f84a134f407fba2477694fdb753e 2023-09-06T20:14:55,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/e278f84a134f407fba2477694fdb753e, entries=150, sequenceid=482, filesize=12.0 K 2023-09-06T20:14:55,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for e66eccf326cb10336c74bdb6873cf7b9 in 57ms, sequenceid=482, compaction requested=true 2023-09-06T20:14:55,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:55,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:14:55,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:55,673 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:55,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:14:55,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:55,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e66eccf326cb10336c74bdb6873cf7b9:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:14:55,673 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:55,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:55,674 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37942 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:55,674 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/A is initiating minor compaction (all files) 2023-09-06T20:14:55,674 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/A in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:55,674 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37942 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:55,674 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d08c0a63ec904c389acc7d7e938aa9a2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/5967c7a2fa0b4410b9feb8f23ba4fdc6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/4c4e0591bf644582b8b3c7e8a73b7eba] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=37.1 K 2023-09-06T20:14:55,674 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/B is initiating minor compaction (all files) 2023-09-06T20:14:55,674 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/B in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:55,674 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/3eff0dd6ffd9450dac82c2777d88748f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/0a794ec6ce3b494cb3bdc79eb0a5030e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/46c1b64942c542c293dadb3c352b9932] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=37.1 K 2023-09-06T20:14:55,675 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting d08c0a63ec904c389acc7d7e938aa9a2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1694031292731 2023-09-06T20:14:55,675 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5967c7a2fa0b4410b9feb8f23ba4fdc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1694031293868 2023-09-06T20:14:55,675 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 3eff0dd6ffd9450dac82c2777d88748f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1694031292731 2023-09-06T20:14:55,676 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4c4e0591bf644582b8b3c7e8a73b7eba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1694031295001 2023-09-06T20:14:55,676 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 0a794ec6ce3b494cb3bdc79eb0a5030e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1694031293868 2023-09-06T20:14:55,676 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 46c1b64942c542c293dadb3c352b9932, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1694031295001 2023-09-06T20:14:55,687 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#A#compaction#325 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:55,687 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/8d26c8e828924135bb9650542b5ce241 is 50, key is test_row_0/A:col10/1694031295001/Put/seqid=0 2023-09-06T20:14:55,689 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#B#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:55,689 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/cd37caf450044d4999381c090a46301d is 50, key is test_row_0/B:col10/1694031295001/Put/seqid=0 2023-09-06T20:14:55,706 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/8d26c8e828924135bb9650542b5ce241 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/8d26c8e828924135bb9650542b5ce241 2023-09-06T20:14:55,711 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/A of e66eccf326cb10336c74bdb6873cf7b9 into 8d26c8e828924135bb9650542b5ce241(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:55,711 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:55,711 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/A, priority=13, startTime=1694031295673; duration=0sec 2023-09-06T20:14:55,711 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:14:55,711 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:A 2023-09-06T20:14:55,711 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:14:55,713 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37942 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:14:55,713 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): e66eccf326cb10336c74bdb6873cf7b9/C is initiating minor compaction (all files) 2023-09-06T20:14:55,713 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of e66eccf326cb10336c74bdb6873cf7b9/C in TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:55,713 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/181d127f87734b8ba39f5bdb202925de, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/896bcc9a6f8e487a85726fba3397b0d2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/e278f84a134f407fba2477694fdb753e] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp, totalSize=37.1 K 2023-09-06T20:14:55,714 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 181d127f87734b8ba39f5bdb202925de, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1694031292731 2023-09-06T20:14:55,714 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 896bcc9a6f8e487a85726fba3397b0d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1694031293868 2023-09-06T20:14:55,714 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting e278f84a134f407fba2477694fdb753e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1694031295001 2023-09-06T20:14:55,722 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e66eccf326cb10336c74bdb6873cf7b9#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:14:55,722 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/9a8aa76735234bd0979391ebb47c9e9c is 50, key is test_row_0/C:col10/1694031295001/Put/seqid=0 2023-09-06T20:14:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:55,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2023-09-06T20:14:55,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:55,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:55,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:55,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:55,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:55,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:55,741 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/9a8aa76735234bd0979391ebb47c9e9c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/9a8aa76735234bd0979391ebb47c9e9c 2023-09-06T20:14:55,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/99ae37f2442447d6b385cc4353010f49 is 50, key is test_row_0/A:col10/1694031295633/Put/seqid=0 2023-09-06T20:14:55,747 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/C of e66eccf326cb10336c74bdb6873cf7b9 into 9a8aa76735234bd0979391ebb47c9e9c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:55,747 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:55,747 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/C, priority=13, startTime=1694031295673; duration=0sec 2023-09-06T20:14:55,747 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:55,747 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:C 2023-09-06T20:14:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031355757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031355758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031355760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031355861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031355861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:55,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031355866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:56,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:56,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031356063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031356064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:56,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031356069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:56,119 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/cd37caf450044d4999381c090a46301d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/cd37caf450044d4999381c090a46301d 2023-09-06T20:14:56,123 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in e66eccf326cb10336c74bdb6873cf7b9/B of e66eccf326cb10336c74bdb6873cf7b9 into cd37caf450044d4999381c090a46301d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:14:56,123 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:56,123 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., storeName=e66eccf326cb10336c74bdb6873cf7b9/B, priority=13, startTime=1694031295673; duration=0sec 2023-09-06T20:14:56,123 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:14:56,123 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e66eccf326cb10336c74bdb6873cf7b9:B 2023-09-06T20:14:56,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/99ae37f2442447d6b385cc4353010f49 2023-09-06T20:14:56,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/971c58ea8df04b2a91c753458818b02d is 50, key is test_row_0/B:col10/1694031295633/Put/seqid=0 2023-09-06T20:14:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-06T20:14:56,216 INFO [Thread-1447 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 95 completed 2023-09-06T20:14:56,217 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:14:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2023-09-06T20:14:56,220 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:14:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-09-06T20:14:56,221 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:14:56,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:14:56,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48114 deadline: 1694031356256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:56,259 DEBUG [Thread-1439 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4149 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9., hostname=jenkins-hbase3.apache.org,33275,1694031217268, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:130) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-06T20:14:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-09-06T20:14:56,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:56,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031356367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:56,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:56,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031356367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:56,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2023-09-06T20:14:56,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031356377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:56,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(53): pid=98 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-09-06T20:14:56,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2023-09-06T20:14:56,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:56,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:56,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:56,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(53): pid=98 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/971c58ea8df04b2a91c753458818b02d 2023-09-06T20:14:56,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/91aeba9d4b664430adb823820cdc3509 is 50, key is test_row_0/C:col10/1694031295633/Put/seqid=0 2023-09-06T20:14:56,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2023-09-06T20:14:56,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:56,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:56,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:56,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(53): pid=98 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-09-06T20:14:56,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2023-09-06T20:14:56,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:56,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:56,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:56,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(53): pid=98 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48132 deadline: 1694031356871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:56,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48120 deadline: 1694031356871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:56,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:14:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:48140 deadline: 1694031356878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:14:56,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2023-09-06T20:14:56,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:56,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:56,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:56,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(53): pid=98 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:14:56,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/91aeba9d4b664430adb823820cdc3509 2023-09-06T20:14:57,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/99ae37f2442447d6b385cc4353010f49 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/99ae37f2442447d6b385cc4353010f49 2023-09-06T20:14:57,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/99ae37f2442447d6b385cc4353010f49, entries=150, sequenceid=504, filesize=12.0 K 2023-09-06T20:14:57,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/971c58ea8df04b2a91c753458818b02d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/971c58ea8df04b2a91c753458818b02d 2023-09-06T20:14:57,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/971c58ea8df04b2a91c753458818b02d, entries=150, sequenceid=504, filesize=12.0 K 2023-09-06T20:14:57,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/91aeba9d4b664430adb823820cdc3509 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/91aeba9d4b664430adb823820cdc3509 2023-09-06T20:14:57,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/91aeba9d4b664430adb823820cdc3509, entries=150, sequenceid=504, filesize=12.0 K 2023-09-06T20:14:57,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for e66eccf326cb10336c74bdb6873cf7b9 in 1273ms, sequenceid=504, compaction requested=false 2023-09-06T20:14:57,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:57,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2023-09-06T20:14:57,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:57,143 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-06T20:14:57,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:14:57,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:57,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:14:57,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:57,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:14:57,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:14:57,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/a464e659030a4a6e854ae37d3b255720 is 50, key is test_row_0/A:col10/1694031295756/Put/seqid=0 2023-09-06T20:14:57,292 DEBUG [Thread-1448 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x38d09399 to 127.0.0.1:58481 2023-09-06T20:14:57,292 DEBUG [Thread-1448 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:57,293 DEBUG [Thread-1450 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1f0aa952 to 127.0.0.1:58481 2023-09-06T20:14:57,293 DEBUG [Thread-1450 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:57,294 DEBUG [Thread-1454 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7b9dc353 to 127.0.0.1:58481 2023-09-06T20:14:57,294 DEBUG [Thread-1454 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:57,294 DEBUG [Thread-1452 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x26513133 to 127.0.0.1:58481 2023-09-06T20:14:57,294 DEBUG [Thread-1452 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:57,294 DEBUG [Thread-1456 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1f084375 to 127.0.0.1:58481 2023-09-06T20:14:57,294 DEBUG [Thread-1456 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-09-06T20:14:57,568 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/a464e659030a4a6e854ae37d3b255720 2023-09-06T20:14:57,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/d4887bf97e73420facfb00c18cb060f4 is 50, key is test_row_0/B:col10/1694031295756/Put/seqid=0 2023-09-06T20:14:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:14:57,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. as already flushing 2023-09-06T20:14:57,881 DEBUG [Thread-1441 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7c931dab to 127.0.0.1:58481 2023-09-06T20:14:57,881 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:57,883 DEBUG [Thread-1445 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x70d13a10 to 127.0.0.1:58481 2023-09-06T20:14:57,883 DEBUG [Thread-1445 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:57,887 DEBUG [Thread-1437 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0e83f4c3 to 127.0.0.1:58481 2023-09-06T20:14:57,887 DEBUG [Thread-1437 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:14:57,977 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/d4887bf97e73420facfb00c18cb060f4 2023-09-06T20:14:57,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/6259bd34a13d46169e947418d7f6f60f is 50, key is test_row_0/C:col10/1694031295756/Put/seqid=0 2023-09-06T20:14:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-09-06T20:14:58,388 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/6259bd34a13d46169e947418d7f6f60f 2023-09-06T20:14:58,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/a464e659030a4a6e854ae37d3b255720 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/a464e659030a4a6e854ae37d3b255720 2023-09-06T20:14:58,395 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/a464e659030a4a6e854ae37d3b255720, entries=150, sequenceid=521, filesize=12.0 K 2023-09-06T20:14:58,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/d4887bf97e73420facfb00c18cb060f4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d4887bf97e73420facfb00c18cb060f4 2023-09-06T20:14:58,399 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d4887bf97e73420facfb00c18cb060f4, entries=150, sequenceid=521, filesize=12.0 K 2023-09-06T20:14:58,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/6259bd34a13d46169e947418d7f6f60f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/6259bd34a13d46169e947418d7f6f60f 2023-09-06T20:14:58,403 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/6259bd34a13d46169e947418d7f6f60f, entries=150, sequenceid=521, filesize=12.0 K 2023-09-06T20:14:58,404 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=20.13 KB/20610 for e66eccf326cb10336c74bdb6873cf7b9 in 1261ms, sequenceid=521, compaction requested=true 2023-09-06T20:14:58,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2520): Flush status journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:14:58,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:14:58,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=98 2023-09-06T20:14:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=98 2023-09-06T20:14:58,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=98, resume processing ppid=97 2023-09-06T20:14:58,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=98, ppid=97, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1840 sec 2023-09-06T20:14:58,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=97, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.1890 sec 2023-09-06T20:15:00,287 DEBUG [Thread-1439 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x65bbaf59 to 127.0.0.1:58481 2023-09-06T20:15:00,287 DEBUG [Thread-1439 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-09-06T20:15:00,325 INFO [Thread-1447 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 97 completed 2023-09-06T20:15:02,243 DEBUG [Thread-1443 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2aba9a6f to 127.0.0.1:58481 2023-09-06T20:15:02,243 DEBUG [Thread-1443 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:02,243 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2023-09-06T20:15:02,243 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2023-09-06T20:15:02,243 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2023-09-06T20:15:02,243 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2023-09-06T20:15:02,243 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2023-09-06T20:15:02,243 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2023-09-06T20:15:02,243 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2023-09-06T20:15:02,243 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(398): read 6959 2023-09-06T20:15:02,243 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(398): read 6879 2023-09-06T20:15:02,243 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(398): read 6907 2023-09-06T20:15:02,243 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(398): read 6969 2023-09-06T20:15:02,243 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(398): read 6871 2023-09-06T20:15:02,243 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2023-09-06T20:15:02,243 INFO [Listener at localhost/35961 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:15:02,243 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x279eab79 to 127.0.0.1:58481 2023-09-06T20:15:02,244 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:02,244 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2023-09-06T20:15:02,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable TestAcidGuarantees 2023-09-06T20:15:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=99, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-06T20:15:02,247 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031302247"}]},"ts":"1694031302247"} 2023-09-06T20:15:02,248 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2023-09-06T20:15:02,250 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2023-09-06T20:15:02,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=e66eccf326cb10336c74bdb6873cf7b9, UNASSIGN}] 2023-09-06T20:15:02,252 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=e66eccf326cb10336c74bdb6873cf7b9, UNASSIGN 2023-09-06T20:15:02,252 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=100 updating hbase:meta row=e66eccf326cb10336c74bdb6873cf7b9, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:02,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE; CloseRegionProcedure e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:15:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-06T20:15:02,405 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(119): Close e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:15:02,405 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1668): Closing e66eccf326cb10336c74bdb6873cf7b9, disabling compactions & flushes 2023-09-06T20:15:02,405 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:15:02,405 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:15:02,405 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. after waiting 0 ms 2023-09-06T20:15:02,405 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:15:02,405 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(2819): Flushing e66eccf326cb10336c74bdb6873cf7b9 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2023-09-06T20:15:02,405 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=A 2023-09-06T20:15:02,405 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:02,405 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=B 2023-09-06T20:15:02,405 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:02,405 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.CompactingMemStore(204): FLUSHING TO DISK e66eccf326cb10336c74bdb6873cf7b9, store=C 2023-09-06T20:15:02,405 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:02,408 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/707990531d7d4f6faf7ee4833357a288 is 50, key is test_row_0/A:col10/1694031297880/Put/seqid=0 2023-09-06T20:15:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-06T20:15:02,813 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/707990531d7d4f6faf7ee4833357a288 2023-09-06T20:15:02,818 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/bcae18d887c74648a8897a8ff24ec0d1 is 50, key is test_row_0/B:col10/1694031297880/Put/seqid=0 2023-09-06T20:15:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-06T20:15:03,222 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/bcae18d887c74648a8897a8ff24ec0d1 2023-09-06T20:15:03,227 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/6bb76e0cab5b4f88b1453e930da69223 is 50, key is test_row_0/C:col10/1694031297880/Put/seqid=0 2023-09-06T20:15:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-06T20:15:03,632 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/6bb76e0cab5b4f88b1453e930da69223 2023-09-06T20:15:03,636 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/A/707990531d7d4f6faf7ee4833357a288 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/707990531d7d4f6faf7ee4833357a288 2023-09-06T20:15:03,639 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/707990531d7d4f6faf7ee4833357a288, entries=150, sequenceid=529, filesize=12.0 K 2023-09-06T20:15:03,639 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/B/bcae18d887c74648a8897a8ff24ec0d1 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/bcae18d887c74648a8897a8ff24ec0d1 2023-09-06T20:15:03,643 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/bcae18d887c74648a8897a8ff24ec0d1, entries=150, sequenceid=529, filesize=12.0 K 2023-09-06T20:15:03,643 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/.tmp/C/6bb76e0cab5b4f88b1453e930da69223 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/6bb76e0cab5b4f88b1453e930da69223 2023-09-06T20:15:03,646 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/6bb76e0cab5b4f88b1453e930da69223, entries=150, sequenceid=529, filesize=12.0 K 2023-09-06T20:15:03,647 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(3022): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e66eccf326cb10336c74bdb6873cf7b9 in 1242ms, sequenceid=529, compaction requested=true 2023-09-06T20:15:03,669 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/f309c3403aef452a94a309fe1b5f9259, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/16ad974fcc9f4e0b95ba68b4066923e0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/fbac69b2d0b04fe5a38d57aedff0d94b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/5784b7790a61475f9eded82518e4537c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/51cb1847abbb4b45a62a3391548d8f96, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/047f17550e964f5e940799fb0c4fcf82, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/180768c30cee47fda356508cea1e9138, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/1d7029007d5e4606a881a2d47d54e72d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/34d242df3f954b5daf7dbbc302d137e5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/f0b7ad9996a64bc199f180d58a9bc84c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/234fa368a45347f3890cceb871bb8134, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b2fb56c9b9134e23a622292e6f2b74ab, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/3d68f96319a146a6be47aaa34cd1d559, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d2fbf02e84144afc9801263339539afd, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/53f4210e0c9840a097c471f5c41a527d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/036d435a627c48768b7e5e99466ff638, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/7d744ffdb22e4656bd144b5bdc7aabe3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d124867072cd4a2b81ca098570ddaee4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/67dba75690674d9b9f9c56e3beb43ab0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/1080f9fc074d4780bd7742086675d2d3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/65921ef378e1423d84b1ba7b0be3bee5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/be3a4519f1284cad90eb9e41b0310c17, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/7ff253ef60994aaeaf02e16450ef6e69, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/fd3ff8a5c84a4de9b3a9e18f356f48a0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/99948f37fe084c28baabb4f8655cea87, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/daf5a5c0920844c6853f0fcba849023b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/479ccee0aef244bfbcd8207ec85e86b7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b8caecdc64c44b09bc119410ea722b18, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/baa207d5524e4f1a95d4b13ec4afaea3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/482cb447a12f47ea9ca840534ea592a7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/6b68ac38e1fa4edb9c102cc6774616b1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d08c0a63ec904c389acc7d7e938aa9a2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b825572bca824048acbb094777173a27, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/5967c7a2fa0b4410b9feb8f23ba4fdc6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/4c4e0591bf644582b8b3c7e8a73b7eba] to archive 2023-09-06T20:15:03,670 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:15:03,671 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/f309c3403aef452a94a309fe1b5f9259 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/f309c3403aef452a94a309fe1b5f9259 2023-09-06T20:15:03,673 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/16ad974fcc9f4e0b95ba68b4066923e0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/16ad974fcc9f4e0b95ba68b4066923e0 2023-09-06T20:15:03,674 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/fbac69b2d0b04fe5a38d57aedff0d94b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/fbac69b2d0b04fe5a38d57aedff0d94b 2023-09-06T20:15:03,675 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/5784b7790a61475f9eded82518e4537c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/5784b7790a61475f9eded82518e4537c 2023-09-06T20:15:03,676 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/51cb1847abbb4b45a62a3391548d8f96 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/51cb1847abbb4b45a62a3391548d8f96 2023-09-06T20:15:03,677 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/047f17550e964f5e940799fb0c4fcf82 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/047f17550e964f5e940799fb0c4fcf82 2023-09-06T20:15:03,678 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/180768c30cee47fda356508cea1e9138 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/180768c30cee47fda356508cea1e9138 2023-09-06T20:15:03,679 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/1d7029007d5e4606a881a2d47d54e72d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/1d7029007d5e4606a881a2d47d54e72d 2023-09-06T20:15:03,680 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/34d242df3f954b5daf7dbbc302d137e5 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/34d242df3f954b5daf7dbbc302d137e5 2023-09-06T20:15:03,681 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/f0b7ad9996a64bc199f180d58a9bc84c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/f0b7ad9996a64bc199f180d58a9bc84c 2023-09-06T20:15:03,682 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/234fa368a45347f3890cceb871bb8134 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/234fa368a45347f3890cceb871bb8134 2023-09-06T20:15:03,683 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b2fb56c9b9134e23a622292e6f2b74ab to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b2fb56c9b9134e23a622292e6f2b74ab 2023-09-06T20:15:03,683 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/3d68f96319a146a6be47aaa34cd1d559 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/3d68f96319a146a6be47aaa34cd1d559 2023-09-06T20:15:03,684 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d2fbf02e84144afc9801263339539afd to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d2fbf02e84144afc9801263339539afd 2023-09-06T20:15:03,685 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/53f4210e0c9840a097c471f5c41a527d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/53f4210e0c9840a097c471f5c41a527d 2023-09-06T20:15:03,687 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/036d435a627c48768b7e5e99466ff638 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/036d435a627c48768b7e5e99466ff638 2023-09-06T20:15:03,688 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/7d744ffdb22e4656bd144b5bdc7aabe3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/7d744ffdb22e4656bd144b5bdc7aabe3 2023-09-06T20:15:03,689 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d124867072cd4a2b81ca098570ddaee4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d124867072cd4a2b81ca098570ddaee4 2023-09-06T20:15:03,690 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/67dba75690674d9b9f9c56e3beb43ab0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/67dba75690674d9b9f9c56e3beb43ab0 2023-09-06T20:15:03,691 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/1080f9fc074d4780bd7742086675d2d3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/1080f9fc074d4780bd7742086675d2d3 2023-09-06T20:15:03,692 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/65921ef378e1423d84b1ba7b0be3bee5 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/65921ef378e1423d84b1ba7b0be3bee5 2023-09-06T20:15:03,693 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/be3a4519f1284cad90eb9e41b0310c17 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/be3a4519f1284cad90eb9e41b0310c17 2023-09-06T20:15:03,694 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/7ff253ef60994aaeaf02e16450ef6e69 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/7ff253ef60994aaeaf02e16450ef6e69 2023-09-06T20:15:03,694 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/fd3ff8a5c84a4de9b3a9e18f356f48a0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/fd3ff8a5c84a4de9b3a9e18f356f48a0 2023-09-06T20:15:03,695 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/99948f37fe084c28baabb4f8655cea87 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/99948f37fe084c28baabb4f8655cea87 2023-09-06T20:15:03,696 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/daf5a5c0920844c6853f0fcba849023b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/daf5a5c0920844c6853f0fcba849023b 2023-09-06T20:15:03,697 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/479ccee0aef244bfbcd8207ec85e86b7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/479ccee0aef244bfbcd8207ec85e86b7 2023-09-06T20:15:03,698 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b8caecdc64c44b09bc119410ea722b18 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b8caecdc64c44b09bc119410ea722b18 2023-09-06T20:15:03,699 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/baa207d5524e4f1a95d4b13ec4afaea3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/baa207d5524e4f1a95d4b13ec4afaea3 2023-09-06T20:15:03,701 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/482cb447a12f47ea9ca840534ea592a7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/482cb447a12f47ea9ca840534ea592a7 2023-09-06T20:15:03,702 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/6b68ac38e1fa4edb9c102cc6774616b1 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/6b68ac38e1fa4edb9c102cc6774616b1 2023-09-06T20:15:03,703 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d08c0a63ec904c389acc7d7e938aa9a2 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/d08c0a63ec904c389acc7d7e938aa9a2 2023-09-06T20:15:03,703 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b825572bca824048acbb094777173a27 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/b825572bca824048acbb094777173a27 2023-09-06T20:15:03,704 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/5967c7a2fa0b4410b9feb8f23ba4fdc6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/5967c7a2fa0b4410b9feb8f23ba4fdc6 2023-09-06T20:15:03,705 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/4c4e0591bf644582b8b3c7e8a73b7eba to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/4c4e0591bf644582b8b3c7e8a73b7eba 2023-09-06T20:15:03,709 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/5fada1f330084a198d229db14b4a7588, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d1df3a825c074283ac982d6973cac546, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/e98480c4e57842f59c32d8067d84311c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fd11eafa85e5444b98db604377a05a20, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/92da2b29796d49568c9ff59afb4b6bdf, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/4e4d190ab7a347c5b524131a410e3df8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/2f6cecef0d104f86bab5ca59097daac7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/02113d7581d14c3db9a525dd0d451aa7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/84f5ef8fd35943278f0d06dae1838e7c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/40422d6b3dd049239053f289b8a6b993, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/99a7ffa5a22b4a70bf8d5e05c6ddf8af, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/51a3189079aa4d0ca3e3054885e5627e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/08512a68ea644f2ca78f56f0a5166162, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a78d2292e5d5408e86452894850b1649, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/c8d850515ede473eae6eef0d0e61648f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/58252bd410ca413da28d725b7e13b98d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8bf81faf82d04229853e9bcacd5d4286, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/36cafd16d7804eecbdb33e54aea7fc26, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/95d56aaede1a42d99e23e30b5421b9ea, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fd0efe15fd70461cade7d79cca7b9301, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/1d6387d3f6da49c5b69739c12a9f0ffd, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8bd012a329d945cd9ad03092000de2d4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/5375f09e8e244b0f92de89fd4eea86d7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8c6f3ac5b04e4b538894bc9ef82311c9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/9719893d2b424528abb5fbf03936f7d7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fc8f5b69d56d452aa606141fbcf1031a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/424b9c137abf40a7b8439b7acf037f2c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/770623d98f9443e5a275c567e9656159, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d42fbefce0af4f7ba21be86f5590ff2b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a8280ecb5156411889312f2c67a78669, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a1675524ec5a46f2ae1dbd15da188c48, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/3eff0dd6ffd9450dac82c2777d88748f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/ba18ef8dc7644642b7e341a07f86fc06, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/0a794ec6ce3b494cb3bdc79eb0a5030e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/46c1b64942c542c293dadb3c352b9932] to archive 2023-09-06T20:15:03,710 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:15:03,711 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/5fada1f330084a198d229db14b4a7588 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/5fada1f330084a198d229db14b4a7588 2023-09-06T20:15:03,712 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d1df3a825c074283ac982d6973cac546 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d1df3a825c074283ac982d6973cac546 2023-09-06T20:15:03,713 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/e98480c4e57842f59c32d8067d84311c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/e98480c4e57842f59c32d8067d84311c 2023-09-06T20:15:03,715 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fd11eafa85e5444b98db604377a05a20 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fd11eafa85e5444b98db604377a05a20 2023-09-06T20:15:03,716 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/92da2b29796d49568c9ff59afb4b6bdf to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/92da2b29796d49568c9ff59afb4b6bdf 2023-09-06T20:15:03,717 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/4e4d190ab7a347c5b524131a410e3df8 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/4e4d190ab7a347c5b524131a410e3df8 2023-09-06T20:15:03,718 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/2f6cecef0d104f86bab5ca59097daac7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/2f6cecef0d104f86bab5ca59097daac7 2023-09-06T20:15:03,719 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/02113d7581d14c3db9a525dd0d451aa7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/02113d7581d14c3db9a525dd0d451aa7 2023-09-06T20:15:03,720 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/84f5ef8fd35943278f0d06dae1838e7c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/84f5ef8fd35943278f0d06dae1838e7c 2023-09-06T20:15:03,721 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/40422d6b3dd049239053f289b8a6b993 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/40422d6b3dd049239053f289b8a6b993 2023-09-06T20:15:03,722 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/99a7ffa5a22b4a70bf8d5e05c6ddf8af to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/99a7ffa5a22b4a70bf8d5e05c6ddf8af 2023-09-06T20:15:03,723 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/51a3189079aa4d0ca3e3054885e5627e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/51a3189079aa4d0ca3e3054885e5627e 2023-09-06T20:15:03,723 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/08512a68ea644f2ca78f56f0a5166162 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/08512a68ea644f2ca78f56f0a5166162 2023-09-06T20:15:03,725 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a78d2292e5d5408e86452894850b1649 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a78d2292e5d5408e86452894850b1649 2023-09-06T20:15:03,726 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/c8d850515ede473eae6eef0d0e61648f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/c8d850515ede473eae6eef0d0e61648f 2023-09-06T20:15:03,727 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/58252bd410ca413da28d725b7e13b98d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/58252bd410ca413da28d725b7e13b98d 2023-09-06T20:15:03,728 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8bf81faf82d04229853e9bcacd5d4286 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8bf81faf82d04229853e9bcacd5d4286 2023-09-06T20:15:03,729 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/36cafd16d7804eecbdb33e54aea7fc26 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/36cafd16d7804eecbdb33e54aea7fc26 2023-09-06T20:15:03,730 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/95d56aaede1a42d99e23e30b5421b9ea to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/95d56aaede1a42d99e23e30b5421b9ea 2023-09-06T20:15:03,731 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fd0efe15fd70461cade7d79cca7b9301 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fd0efe15fd70461cade7d79cca7b9301 2023-09-06T20:15:03,732 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/1d6387d3f6da49c5b69739c12a9f0ffd to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/1d6387d3f6da49c5b69739c12a9f0ffd 2023-09-06T20:15:03,733 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8bd012a329d945cd9ad03092000de2d4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8bd012a329d945cd9ad03092000de2d4 2023-09-06T20:15:03,734 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/5375f09e8e244b0f92de89fd4eea86d7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/5375f09e8e244b0f92de89fd4eea86d7 2023-09-06T20:15:03,735 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8c6f3ac5b04e4b538894bc9ef82311c9 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/8c6f3ac5b04e4b538894bc9ef82311c9 2023-09-06T20:15:03,735 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/9719893d2b424528abb5fbf03936f7d7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/9719893d2b424528abb5fbf03936f7d7 2023-09-06T20:15:03,736 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fc8f5b69d56d452aa606141fbcf1031a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/fc8f5b69d56d452aa606141fbcf1031a 2023-09-06T20:15:03,737 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/424b9c137abf40a7b8439b7acf037f2c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/424b9c137abf40a7b8439b7acf037f2c 2023-09-06T20:15:03,739 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/770623d98f9443e5a275c567e9656159 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/770623d98f9443e5a275c567e9656159 2023-09-06T20:15:03,740 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d42fbefce0af4f7ba21be86f5590ff2b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d42fbefce0af4f7ba21be86f5590ff2b 2023-09-06T20:15:03,740 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a8280ecb5156411889312f2c67a78669 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a8280ecb5156411889312f2c67a78669 2023-09-06T20:15:03,741 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a1675524ec5a46f2ae1dbd15da188c48 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/a1675524ec5a46f2ae1dbd15da188c48 2023-09-06T20:15:03,742 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/3eff0dd6ffd9450dac82c2777d88748f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/3eff0dd6ffd9450dac82c2777d88748f 2023-09-06T20:15:03,743 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/ba18ef8dc7644642b7e341a07f86fc06 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/ba18ef8dc7644642b7e341a07f86fc06 2023-09-06T20:15:03,744 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/0a794ec6ce3b494cb3bdc79eb0a5030e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/0a794ec6ce3b494cb3bdc79eb0a5030e 2023-09-06T20:15:03,745 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/46c1b64942c542c293dadb3c352b9932 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/46c1b64942c542c293dadb3c352b9932 2023-09-06T20:15:03,748 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/fd9373a3fcc848dc862a4cab3dca2cf3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/8abfc13fcff14256ba2068fdcc34c90e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/520045e0bde44a1f81f5e6e0409b0c0a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/a342a434aa4a4ada89d303b5faf33989, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/de3983ff99ce43b1ab9157c6663c53d7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/cc22c4bccfa2432d883f0961d0c88ed3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/df5ca179b03e4d2c8e11348f7fa1f89c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/6fa386f2d22d4dae802d9823e36cada7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/0b04dde83edf4925876026457bb13749, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/61f5c45694c54789a0c182c2c1a7fec1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/960104e7e85e49ffb4217aa9348722c6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/2e239b5d1ab040899619d5a3dc3245a9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/d31fc018e1964bf8b26bdd55dff8155b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/1429b807c3cf491080c1feb322947354, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/ddab64a23b3a43fe9cb0730b421c5ffe, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/25355f9d38de42fc8d95dc63e2af23e8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/79e97478a7224acdbdf222d47963e827, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/ae46ee0300dd41f6b35f370b3c105699, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/5e7dc257a48543adb350660212e216ab, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/1dc8c2bc27c5492ab3bfae4d8e402017, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/11272f9010274fbeb34c54ce9d86362d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/02469cad4b584f7d82d98fea94274560, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/52628cb37e9d401cb4481f1e4875da6e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/147fb2985937456a8c55d017f212196e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/a123a9e1bff143d3b6ae7946d36f963b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/acd78159a8fe441f9c01ad9482f4de3e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/d57766b1948a416d9588fe461f80d5c8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/c49f1e9d26e644f6a1d0bc60db9530e0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/3978b65a6a2445d69e1e061bd70da9ce, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/acffe2fe15b0474b89983af52c6a6d5c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/8617eeb41a494fd9a85f3c5bf09eab4d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/181d127f87734b8ba39f5bdb202925de, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/b145935ef8a446fb8899d48e8f5d34b1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/896bcc9a6f8e487a85726fba3397b0d2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/e278f84a134f407fba2477694fdb753e] to archive 2023-09-06T20:15:03,749 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:15:03,750 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/fd9373a3fcc848dc862a4cab3dca2cf3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/fd9373a3fcc848dc862a4cab3dca2cf3 2023-09-06T20:15:03,751 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/8abfc13fcff14256ba2068fdcc34c90e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/8abfc13fcff14256ba2068fdcc34c90e 2023-09-06T20:15:03,752 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/520045e0bde44a1f81f5e6e0409b0c0a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/520045e0bde44a1f81f5e6e0409b0c0a 2023-09-06T20:15:03,753 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/a342a434aa4a4ada89d303b5faf33989 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/a342a434aa4a4ada89d303b5faf33989 2023-09-06T20:15:03,754 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/de3983ff99ce43b1ab9157c6663c53d7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/de3983ff99ce43b1ab9157c6663c53d7 2023-09-06T20:15:03,755 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/cc22c4bccfa2432d883f0961d0c88ed3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/cc22c4bccfa2432d883f0961d0c88ed3 2023-09-06T20:15:03,756 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/df5ca179b03e4d2c8e11348f7fa1f89c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/df5ca179b03e4d2c8e11348f7fa1f89c 2023-09-06T20:15:03,756 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/6fa386f2d22d4dae802d9823e36cada7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/6fa386f2d22d4dae802d9823e36cada7 2023-09-06T20:15:03,757 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/0b04dde83edf4925876026457bb13749 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/0b04dde83edf4925876026457bb13749 2023-09-06T20:15:03,759 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/61f5c45694c54789a0c182c2c1a7fec1 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/61f5c45694c54789a0c182c2c1a7fec1 2023-09-06T20:15:03,760 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/960104e7e85e49ffb4217aa9348722c6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/960104e7e85e49ffb4217aa9348722c6 2023-09-06T20:15:03,761 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/2e239b5d1ab040899619d5a3dc3245a9 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/2e239b5d1ab040899619d5a3dc3245a9 2023-09-06T20:15:03,762 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/d31fc018e1964bf8b26bdd55dff8155b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/d31fc018e1964bf8b26bdd55dff8155b 2023-09-06T20:15:03,763 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/1429b807c3cf491080c1feb322947354 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/1429b807c3cf491080c1feb322947354 2023-09-06T20:15:03,764 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/ddab64a23b3a43fe9cb0730b421c5ffe to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/ddab64a23b3a43fe9cb0730b421c5ffe 2023-09-06T20:15:03,765 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/25355f9d38de42fc8d95dc63e2af23e8 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/25355f9d38de42fc8d95dc63e2af23e8 2023-09-06T20:15:03,765 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/79e97478a7224acdbdf222d47963e827 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/79e97478a7224acdbdf222d47963e827 2023-09-06T20:15:03,766 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/ae46ee0300dd41f6b35f370b3c105699 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/ae46ee0300dd41f6b35f370b3c105699 2023-09-06T20:15:03,767 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/5e7dc257a48543adb350660212e216ab to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/5e7dc257a48543adb350660212e216ab 2023-09-06T20:15:03,768 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/1dc8c2bc27c5492ab3bfae4d8e402017 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/1dc8c2bc27c5492ab3bfae4d8e402017 2023-09-06T20:15:03,769 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/11272f9010274fbeb34c54ce9d86362d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/11272f9010274fbeb34c54ce9d86362d 2023-09-06T20:15:03,771 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/02469cad4b584f7d82d98fea94274560 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/02469cad4b584f7d82d98fea94274560 2023-09-06T20:15:03,772 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/52628cb37e9d401cb4481f1e4875da6e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/52628cb37e9d401cb4481f1e4875da6e 2023-09-06T20:15:03,773 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/147fb2985937456a8c55d017f212196e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/147fb2985937456a8c55d017f212196e 2023-09-06T20:15:03,774 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/a123a9e1bff143d3b6ae7946d36f963b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/a123a9e1bff143d3b6ae7946d36f963b 2023-09-06T20:15:03,775 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/acd78159a8fe441f9c01ad9482f4de3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/acd78159a8fe441f9c01ad9482f4de3e 2023-09-06T20:15:03,775 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/d57766b1948a416d9588fe461f80d5c8 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/d57766b1948a416d9588fe461f80d5c8 2023-09-06T20:15:03,776 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/c49f1e9d26e644f6a1d0bc60db9530e0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/c49f1e9d26e644f6a1d0bc60db9530e0 2023-09-06T20:15:03,777 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/3978b65a6a2445d69e1e061bd70da9ce to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/3978b65a6a2445d69e1e061bd70da9ce 2023-09-06T20:15:03,778 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/acffe2fe15b0474b89983af52c6a6d5c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/acffe2fe15b0474b89983af52c6a6d5c 2023-09-06T20:15:03,779 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/8617eeb41a494fd9a85f3c5bf09eab4d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/8617eeb41a494fd9a85f3c5bf09eab4d 2023-09-06T20:15:03,780 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/181d127f87734b8ba39f5bdb202925de to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/181d127f87734b8ba39f5bdb202925de 2023-09-06T20:15:03,782 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/b145935ef8a446fb8899d48e8f5d34b1 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/b145935ef8a446fb8899d48e8f5d34b1 2023-09-06T20:15:03,783 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/896bcc9a6f8e487a85726fba3397b0d2 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/896bcc9a6f8e487a85726fba3397b0d2 2023-09-06T20:15:03,784 DEBUG [StoreCloser-TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/e278f84a134f407fba2477694fdb753e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/e278f84a134f407fba2477694fdb753e 2023-09-06T20:15:03,787 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/recovered.edits/532.seqid, newMaxSeqId=532, maxSeqId=1 2023-09-06T20:15:03,789 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9. 2023-09-06T20:15:03,789 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1622): Region close journal for e66eccf326cb10336c74bdb6873cf7b9: 2023-09-06T20:15:03,790 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(163): Closed e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:15:03,790 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=100 updating hbase:meta row=e66eccf326cb10336c74bdb6873cf7b9, regionState=CLOSED 2023-09-06T20:15:03,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=101, resume processing ppid=100 2023-09-06T20:15:03,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=101, ppid=100, state=SUCCESS; CloseRegionProcedure e66eccf326cb10336c74bdb6873cf7b9, server=jenkins-hbase3.apache.org,33275,1694031217268 in 1.5380 sec 2023-09-06T20:15:03,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=100, resume processing ppid=99 2023-09-06T20:15:03,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=100, ppid=99, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=e66eccf326cb10336c74bdb6873cf7b9, UNASSIGN in 1.5420 sec 2023-09-06T20:15:03,794 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031303794"}]},"ts":"1694031303794"} 2023-09-06T20:15:03,795 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2023-09-06T20:15:03,797 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2023-09-06T20:15:03,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=99, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.5520 sec 2023-09-06T20:15:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-06T20:15:04,352 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 99 completed 2023-09-06T20:15:04,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete TestAcidGuarantees 2023-09-06T20:15:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=102, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:04,354 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=102, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=102 2023-09-06T20:15:04,355 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=102, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:04,358 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:15:04,360 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/recovered.edits] 2023-09-06T20:15:04,363 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/707990531d7d4f6faf7ee4833357a288 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/707990531d7d4f6faf7ee4833357a288 2023-09-06T20:15:04,364 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/8d26c8e828924135bb9650542b5ce241 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/8d26c8e828924135bb9650542b5ce241 2023-09-06T20:15:04,365 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/99ae37f2442447d6b385cc4353010f49 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/99ae37f2442447d6b385cc4353010f49 2023-09-06T20:15:04,366 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/a464e659030a4a6e854ae37d3b255720 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/A/a464e659030a4a6e854ae37d3b255720 2023-09-06T20:15:04,368 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/971c58ea8df04b2a91c753458818b02d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/971c58ea8df04b2a91c753458818b02d 2023-09-06T20:15:04,370 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/bcae18d887c74648a8897a8ff24ec0d1 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/bcae18d887c74648a8897a8ff24ec0d1 2023-09-06T20:15:04,372 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/cd37caf450044d4999381c090a46301d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/cd37caf450044d4999381c090a46301d 2023-09-06T20:15:04,373 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d4887bf97e73420facfb00c18cb060f4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/B/d4887bf97e73420facfb00c18cb060f4 2023-09-06T20:15:04,375 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/6259bd34a13d46169e947418d7f6f60f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/6259bd34a13d46169e947418d7f6f60f 2023-09-06T20:15:04,376 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/6bb76e0cab5b4f88b1453e930da69223 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/6bb76e0cab5b4f88b1453e930da69223 2023-09-06T20:15:04,377 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/91aeba9d4b664430adb823820cdc3509 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/91aeba9d4b664430adb823820cdc3509 2023-09-06T20:15:04,378 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/9a8aa76735234bd0979391ebb47c9e9c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/C/9a8aa76735234bd0979391ebb47c9e9c 2023-09-06T20:15:04,380 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/recovered.edits/532.seqid to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9/recovered.edits/532.seqid 2023-09-06T20:15:04,380 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/e66eccf326cb10336c74bdb6873cf7b9 2023-09-06T20:15:04,380 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2023-09-06T20:15:04,383 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=102, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:04,386 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2023-09-06T20:15:04,388 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2023-09-06T20:15:04,389 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=102, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:04,389 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2023-09-06T20:15:04,389 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694031304389"}]},"ts":"9223372036854775807"} 2023-09-06T20:15:04,391 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:15:04,391 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => e66eccf326cb10336c74bdb6873cf7b9, NAME => 'TestAcidGuarantees,,1694031275083.e66eccf326cb10336c74bdb6873cf7b9.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:15:04,391 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2023-09-06T20:15:04,391 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694031304391"}]},"ts":"9223372036854775807"} 2023-09-06T20:15:04,392 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2023-09-06T20:15:04,394 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(127): Finished pid=102, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:04,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=102, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2023-09-06T20:15:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=102 2023-09-06T20:15:04,456 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 102 completed 2023-09-06T20:15:04,468 INFO [Listener at localhost/35961 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testGetAtomicity Thread=219 (was 221), OpenFileDescriptor=599 (was 576) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=509 (was 559), ProcessCount=167 (was 170), AvailableMemoryMB=4954 (was 5036) 2023-09-06T20:15:04,480 INFO [Listener at localhost/35961 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testMobScanAtomicity Thread=219, OpenFileDescriptor=599, MaxFileDescriptor=60000, SystemLoadAverage=509, ProcessCount=167, AvailableMemoryMB=4953 2023-09-06T20:15:04,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-06T20:15:04,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:15:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=103, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:04,484 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=103, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:15:04,484 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:04,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 103 2023-09-06T20:15:04,485 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=103, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:15:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=103 2023-09-06T20:15:04,493 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236 2023-09-06T20:15:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=103 2023-09-06T20:15:04,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=103 2023-09-06T20:15:04,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:15:04,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1668): Closing 3bb591e2a9ad6f7c9630808320336906, disabling compactions & flushes 2023-09-06T20:15:04,901 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:04,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:04,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. after waiting 0 ms 2023-09-06T20:15:04,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:04,901 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:04,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:04,902 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=103, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:15:04,903 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1694031304902"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694031304902"}]},"ts":"1694031304902"} 2023-09-06T20:15:04,904 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:15:04,904 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=103, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:15:04,905 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031304904"}]},"ts":"1694031304904"} 2023-09-06T20:15:04,905 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2023-09-06T20:15:04,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bb591e2a9ad6f7c9630808320336906, ASSIGN}] 2023-09-06T20:15:04,913 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bb591e2a9ad6f7c9630808320336906, ASSIGN 2023-09-06T20:15:04,913 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bb591e2a9ad6f7c9630808320336906, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33275,1694031217268; forceNewPlan=false, retain=false 2023-09-06T20:15:05,064 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=104 updating hbase:meta row=3bb591e2a9ad6f7c9630808320336906, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:05,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE; OpenRegionProcedure 3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:15:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=103 2023-09-06T20:15:05,220 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:05,220 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(7254): Opening region: {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:15:05,220 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:05,221 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:15:05,221 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(7296): checking encryption for 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:05,221 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(7299): checking classloading for 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:05,222 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:05,223 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:05,223 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bb591e2a9ad6f7c9630808320336906 columnFamilyName A 2023-09-06T20:15:05,223 DEBUG [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:05,223 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.HStore(324): Store=3bb591e2a9ad6f7c9630808320336906/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:05,223 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:05,224 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:05,224 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bb591e2a9ad6f7c9630808320336906 columnFamilyName B 2023-09-06T20:15:05,224 DEBUG [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:05,225 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.HStore(324): Store=3bb591e2a9ad6f7c9630808320336906/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:05,225 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:05,225 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:05,225 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bb591e2a9ad6f7c9630808320336906 columnFamilyName C 2023-09-06T20:15:05,225 DEBUG [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:05,226 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.HStore(324): Store=3bb591e2a9ad6f7c9630808320336906/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:05,226 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:05,227 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:05,227 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:05,229 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-06T20:15:05,230 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(1076): writing seq id for 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:05,231 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:15:05,231 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(1093): Opened 3bb591e2a9ad6f7c9630808320336906; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64030654, jitterRate=-0.045868903398513794}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-06T20:15:05,233 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegion(992): Region open journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:05,233 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., pid=105, masterSystemTime=1694031305217 2023-09-06T20:15:05,234 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:05,235 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=105}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:05,235 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=104 updating hbase:meta row=3bb591e2a9ad6f7c9630808320336906, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:05,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=105, resume processing ppid=104 2023-09-06T20:15:05,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=105, ppid=104, state=SUCCESS; OpenRegionProcedure 3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 in 171 msec 2023-09-06T20:15:05,238 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=104, resume processing ppid=103 2023-09-06T20:15:05,238 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=104, ppid=103, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bb591e2a9ad6f7c9630808320336906, ASSIGN in 326 msec 2023-09-06T20:15:05,239 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=103, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:15:05,239 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031305239"}]},"ts":"1694031305239"} 2023-09-06T20:15:05,240 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2023-09-06T20:15:05,243 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=103, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:15:05,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=103, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 761 msec 2023-09-06T20:15:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=103 2023-09-06T20:15:05,589 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 103 completed 2023-09-06T20:15:05,590 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x41c47eb6 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:05,595 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f1b887b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:05,597 DEBUG [Listener at localhost/35961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:05,598 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46312, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:05,599 DEBUG [Listener at localhost/35961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:15:05,601 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34550, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:15:05,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-06T20:15:05,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$13(2700): Client=jenkins//172.31.12.81 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:15:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=106, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:06,015 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.963 2023-09-06T20:15:06,015 INFO [PEWorker-1 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.999 2023-09-06T20:15:06,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2023-09-06T20:15:06,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bb591e2a9ad6f7c9630808320336906, REOPEN/MOVE}] 2023-09-06T20:15:06,020 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bb591e2a9ad6f7c9630808320336906, REOPEN/MOVE 2023-09-06T20:15:06,020 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=3bb591e2a9ad6f7c9630808320336906, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; CloseRegionProcedure 3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:15:06,172 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] handler.UnassignRegionHandler(119): Close 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:06,173 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1668): Closing 3bb591e2a9ad6f7c9630808320336906, disabling compactions & flushes 2023-09-06T20:15:06,173 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:06,173 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:06,173 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. after waiting 0 ms 2023-09-06T20:15:06,173 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:06,176 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:15:06,177 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:06,177 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1622): Region close journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:06,177 WARN [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegionServer(3735): Not adding moved region record: 3bb591e2a9ad6f7c9630808320336906 to self. 2023-09-06T20:15:06,178 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] handler.UnassignRegionHandler(163): Closed 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:06,178 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=3bb591e2a9ad6f7c9630808320336906, regionState=CLOSED 2023-09-06T20:15:06,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=109, resume processing ppid=108 2023-09-06T20:15:06,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=109, ppid=108, state=SUCCESS; CloseRegionProcedure 3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 in 158 msec 2023-09-06T20:15:06,181 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bb591e2a9ad6f7c9630808320336906, REOPEN/MOVE; state=CLOSED, location=jenkins-hbase3.apache.org,33275,1694031217268; forceNewPlan=false, retain=true 2023-09-06T20:15:06,331 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=3bb591e2a9ad6f7c9630808320336906, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=110, ppid=108, state=RUNNABLE; OpenRegionProcedure 3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:15:06,487 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:06,487 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7254): Opening region: {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:15:06,488 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:06,488 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:15:06,488 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7296): checking encryption for 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:06,488 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7299): checking classloading for 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:06,489 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:06,490 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:06,490 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bb591e2a9ad6f7c9630808320336906 columnFamilyName A 2023-09-06T20:15:06,491 DEBUG [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:06,492 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.HStore(324): Store=3bb591e2a9ad6f7c9630808320336906/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:06,492 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:06,493 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:06,493 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bb591e2a9ad6f7c9630808320336906 columnFamilyName B 2023-09-06T20:15:06,493 DEBUG [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:06,493 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.HStore(324): Store=3bb591e2a9ad6f7c9630808320336906/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:06,493 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:06,494 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:06,494 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bb591e2a9ad6f7c9630808320336906 columnFamilyName C 2023-09-06T20:15:06,494 DEBUG [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:06,494 INFO [StoreOpener-3bb591e2a9ad6f7c9630808320336906-1 {}] regionserver.HStore(324): Store=3bb591e2a9ad6f7c9630808320336906/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:06,494 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:06,495 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:06,496 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:06,497 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-06T20:15:06,498 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1076): writing seq id for 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:06,499 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1093): Opened 3bb591e2a9ad6f7c9630808320336906; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74075149, jitterRate=0.10380573570728302}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-06T20:15:06,500 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(992): Region open journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:06,501 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., pid=110, masterSystemTime=1694031306484 2023-09-06T20:15:06,502 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:06,502 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:06,502 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=3bb591e2a9ad6f7c9630808320336906, regionState=OPEN, openSeqNum=5, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=110, resume processing ppid=108 2023-09-06T20:15:06,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=110, ppid=108, state=SUCCESS; OpenRegionProcedure 3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 in 171 msec 2023-09-06T20:15:06,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=108, resume processing ppid=107 2023-09-06T20:15:06,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=108, ppid=107, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bb591e2a9ad6f7c9630808320336906, REOPEN/MOVE in 485 msec 2023-09-06T20:15:06,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=107, resume processing ppid=106 2023-09-06T20:15:06,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=107, ppid=106, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2023-09-06T20:15:06,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=106, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 904 msec 2023-09-06T20:15:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=106 2023-09-06T20:15:06,511 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x278f4124 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:06,520 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45dcbaad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:06,521 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2e6d9e88 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:06,528 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b67c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:06,529 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x48325705 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:06,537 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae91955, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:06,538 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x64069dd3 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:06,542 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e567429, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:06,542 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4fe89067 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:06,549 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd0744c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:06,550 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5a3691cd to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:06,555 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4658ecd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:06,556 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x62a48015 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:06,562 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c72dd75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:06,563 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x08e0c80d to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:06,569 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e82552c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:06,570 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x359fa346 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:06,574 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bb1b558, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:06,575 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3e197663 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:06,579 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d46775, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:06,586 DEBUG [hconnection-0x60294e4f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:06,586 DEBUG [hconnection-0x4f3a05cf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:06,592 DEBUG [hconnection-0x65e3ff12-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:06,592 DEBUG [hconnection-0x245ab292-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:06,592 DEBUG [hconnection-0x1c2d9fda-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:06,592 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46320, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:06,592 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46322, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:06,593 DEBUG [hconnection-0x5f1c20f0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:06,593 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46334, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:06,593 DEBUG [hconnection-0x5cf5cb5a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:06,593 DEBUG [hconnection-0x39f62d4-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:06,594 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46354, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:06,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:06,596 DEBUG [hconnection-0x6ed64d9f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:06,598 DEBUG [hconnection-0x518132b2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:06,599 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46340, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:06,599 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46350, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:06,599 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46374, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:06,599 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46362, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:06,601 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46376, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2023-09-06T20:15:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-06T20:15:06,606 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46384, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:06,608 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:06,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:15:06,629 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:06,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:06,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:06,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:06,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:06,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:06,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:06,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:06,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031366648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031366650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031366649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031366650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031366651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906553e332f3a36400f982f880fbcc34203_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031306619/Put/seqid=0 2023-09-06T20:15:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-06T20:15:06,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031366752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031366752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031366752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031366752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031366753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-06T20:15:06,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:06,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:06,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:06,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(53): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:06,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-06T20:15:06,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-06T20:15:06,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:06,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:06,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:06,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(53): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:06,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:06,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031366955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031366955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031366956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031366957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:06,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031366957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,071 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906553e332f3a36400f982f880fbcc34203_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906553e332f3a36400f982f880fbcc34203_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:07,072 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/be7ea0f3937e4b42816582fdea17ad25, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:07,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/be7ea0f3937e4b42816582fdea17ad25 is 175, key is test_row_0/A:col10/1694031306619/Put/seqid=0 2023-09-06T20:15:07,084 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=19, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/be7ea0f3937e4b42816582fdea17ad25 2023-09-06T20:15:07,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-06T20:15:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:07,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(53): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:07,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/401c069744d7450b9b128129fe5eb280 is 50, key is test_row_0/B:col10/1694031306619/Put/seqid=0 2023-09-06T20:15:07,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/401c069744d7450b9b128129fe5eb280 2023-09-06T20:15:07,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/954b3a9bb66d4930ac5f3583ac717b3a is 50, key is test_row_0/C:col10/1694031306619/Put/seqid=0 2023-09-06T20:15:07,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/954b3a9bb66d4930ac5f3583ac717b3a 2023-09-06T20:15:07,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/be7ea0f3937e4b42816582fdea17ad25 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/be7ea0f3937e4b42816582fdea17ad25 2023-09-06T20:15:07,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/be7ea0f3937e4b42816582fdea17ad25, entries=150, sequenceid=19, filesize=30.2 K 2023-09-06T20:15:07,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/401c069744d7450b9b128129fe5eb280 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/401c069744d7450b9b128129fe5eb280 2023-09-06T20:15:07,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/401c069744d7450b9b128129fe5eb280, entries=150, sequenceid=19, filesize=11.7 K 2023-09-06T20:15:07,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/954b3a9bb66d4930ac5f3583ac717b3a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/954b3a9bb66d4930ac5f3583ac717b3a 2023-09-06T20:15:07,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/954b3a9bb66d4930ac5f3583ac717b3a, entries=150, sequenceid=19, filesize=11.7 K 2023-09-06T20:15:07,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 3bb591e2a9ad6f7c9630808320336906 in 538ms, sequenceid=19, compaction requested=false 2023-09-06T20:15:07,160 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2023-09-06T20:15:07,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-06T20:15:07,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-06T20:15:07,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:07,249 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2023-09-06T20:15:07,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:07,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:07,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:07,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:07,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:07,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:07,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906e69dd326963c4aa1b0efbcea0fdcac28_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031306649/Put/seqid=0 2023-09-06T20:15:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:07,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:07,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031367272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031367272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031367272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031367273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031367274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031367378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031367378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031367379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031367379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031367379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031367582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031367583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031367583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031367584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031367584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,667 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906e69dd326963c4aa1b0efbcea0fdcac28_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906e69dd326963c4aa1b0efbcea0fdcac28_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:07,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/07c4e6a7deda4758b7e1d7674c5100a4, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:07,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/07c4e6a7deda4758b7e1d7674c5100a4 is 175, key is test_row_0/A:col10/1694031306649/Put/seqid=0 2023-09-06T20:15:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-06T20:15:07,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031367887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031367887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031367888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031367889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:07,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:07,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031367890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:08,073 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/07c4e6a7deda4758b7e1d7674c5100a4 2023-09-06T20:15:08,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/01cb204792e74a14a5c28542092f3c51 is 50, key is test_row_0/B:col10/1694031306649/Put/seqid=0 2023-09-06T20:15:08,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031368395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:08,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031368396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:08,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031368396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:08,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031368397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:08,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031368397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:08,495 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/01cb204792e74a14a5c28542092f3c51 2023-09-06T20:15:08,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/bdbe6debf3504c34b1076b790d961106 is 50, key is test_row_0/C:col10/1694031306649/Put/seqid=0 2023-09-06T20:15:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-06T20:15:08,918 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/bdbe6debf3504c34b1076b790d961106 2023-09-06T20:15:08,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/07c4e6a7deda4758b7e1d7674c5100a4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/07c4e6a7deda4758b7e1d7674c5100a4 2023-09-06T20:15:08,927 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/07c4e6a7deda4758b7e1d7674c5100a4, entries=150, sequenceid=41, filesize=30.2 K 2023-09-06T20:15:08,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/01cb204792e74a14a5c28542092f3c51 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/01cb204792e74a14a5c28542092f3c51 2023-09-06T20:15:08,935 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/01cb204792e74a14a5c28542092f3c51, entries=150, sequenceid=41, filesize=11.7 K 2023-09-06T20:15:08,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/bdbe6debf3504c34b1076b790d961106 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/bdbe6debf3504c34b1076b790d961106 2023-09-06T20:15:08,940 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/bdbe6debf3504c34b1076b790d961106, entries=150, sequenceid=41, filesize=11.7 K 2023-09-06T20:15:08,941 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3022): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 3bb591e2a9ad6f7c9630808320336906 in 1692ms, sequenceid=41, compaction requested=false 2023-09-06T20:15:08,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:08,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:08,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=112 2023-09-06T20:15:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=112 2023-09-06T20:15:08,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=112, resume processing ppid=111 2023-09-06T20:15:08,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=112, ppid=111, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3130 sec 2023-09-06T20:15:08,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=111, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.3480 sec 2023-09-06T20:15:09,324 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-06T20:15:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:09,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-06T20:15:09,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:09,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:09,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:09,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:09,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:09,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:09,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090605b1e2c3b27d4a609de4c403cb64a25d_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031309404/Put/seqid=0 2023-09-06T20:15:09,423 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090605b1e2c3b27d4a609de4c403cb64a25d_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090605b1e2c3b27d4a609de4c403cb64a25d_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:09,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/1e83aecc549e4998a74a26f8525a5301, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:09,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/1e83aecc549e4998a74a26f8525a5301 is 175, key is test_row_0/A:col10/1694031309404/Put/seqid=0 2023-09-06T20:15:09,429 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=57, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/1e83aecc549e4998a74a26f8525a5301 2023-09-06T20:15:09,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031369425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031369427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031369428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031369428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031369429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/7f34175be50f4d2eb4f9086290eccf2c is 50, key is test_row_0/B:col10/1694031309404/Put/seqid=0 2023-09-06T20:15:09,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031369532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031369533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031369535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031369535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031369536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031369735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031369738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031369739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031369739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031369740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:09,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/7f34175be50f4d2eb4f9086290eccf2c 2023-09-06T20:15:09,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/0661da3230f34d3faf6047111114679a is 50, key is test_row_0/C:col10/1694031309404/Put/seqid=0 2023-09-06T20:15:10,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031370040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031370041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031370043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031370044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031370046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/0661da3230f34d3faf6047111114679a 2023-09-06T20:15:10,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/1e83aecc549e4998a74a26f8525a5301 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/1e83aecc549e4998a74a26f8525a5301 2023-09-06T20:15:10,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/1e83aecc549e4998a74a26f8525a5301, entries=250, sequenceid=57, filesize=47.0 K 2023-09-06T20:15:10,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/7f34175be50f4d2eb4f9086290eccf2c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/7f34175be50f4d2eb4f9086290eccf2c 2023-09-06T20:15:10,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/7f34175be50f4d2eb4f9086290eccf2c, entries=150, sequenceid=57, filesize=11.7 K 2023-09-06T20:15:10,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/0661da3230f34d3faf6047111114679a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/0661da3230f34d3faf6047111114679a 2023-09-06T20:15:10,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/0661da3230f34d3faf6047111114679a, entries=150, sequenceid=57, filesize=11.7 K 2023-09-06T20:15:10,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 3bb591e2a9ad6f7c9630808320336906 in 885ms, sequenceid=57, compaction requested=true 2023-09-06T20:15:10,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:10,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:10,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:10,290 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:10,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:10,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:10,290 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:10,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:10,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:10,292 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 109998 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:10,292 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/A is initiating minor compaction (all files) 2023-09-06T20:15:10,292 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/A in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:10,292 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/be7ea0f3937e4b42816582fdea17ad25, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/07c4e6a7deda4758b7e1d7674c5100a4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/1e83aecc549e4998a74a26f8525a5301] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=107.4 K 2023-09-06T20:15:10,292 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:10,292 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/be7ea0f3937e4b42816582fdea17ad25, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/07c4e6a7deda4758b7e1d7674c5100a4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/1e83aecc549e4998a74a26f8525a5301] 2023-09-06T20:15:10,293 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:10,293 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/B is initiating minor compaction (all files) 2023-09-06T20:15:10,293 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/B in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:10,293 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/401c069744d7450b9b128129fe5eb280, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/01cb204792e74a14a5c28542092f3c51, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/7f34175be50f4d2eb4f9086290eccf2c] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=35.1 K 2023-09-06T20:15:10,294 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting be7ea0f3937e4b42816582fdea17ad25, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1694031306619 2023-09-06T20:15:10,294 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 401c069744d7450b9b128129fe5eb280, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1694031306619 2023-09-06T20:15:10,294 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 07c4e6a7deda4758b7e1d7674c5100a4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1694031306646 2023-09-06T20:15:10,295 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 01cb204792e74a14a5c28542092f3c51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1694031306646 2023-09-06T20:15:10,295 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1e83aecc549e4998a74a26f8525a5301, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1694031307270 2023-09-06T20:15:10,295 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 7f34175be50f4d2eb4f9086290eccf2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1694031307272 2023-09-06T20:15:10,307 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:10,309 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#B#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:10,310 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/e61736ad7a484bc4b3df6cd9fc88f0f4 is 50, key is test_row_0/B:col10/1694031309404/Put/seqid=0 2023-09-06T20:15:10,318 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230906145b2fae43d74559bd283ae9befa4e66_3bb591e2a9ad6f7c9630808320336906 store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:10,319 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230906145b2fae43d74559bd283ae9befa4e66_3bb591e2a9ad6f7c9630808320336906, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:10,319 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906145b2fae43d74559bd283ae9befa4e66_3bb591e2a9ad6f7c9630808320336906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:10,346 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#A#compaction#346 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:10,346 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/9b2beb5435044c91b57b8ab617c5786b is 175, key is test_row_0/A:col10/1694031309404/Put/seqid=0 2023-09-06T20:15:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:10,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2023-09-06T20:15:10,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:10,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:10,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:10,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:10,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:10,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:10,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309069d6bd64b595f4de2a59b5b5cfff03f29_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031309423/Put/seqid=0 2023-09-06T20:15:10,564 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309069d6bd64b595f4de2a59b5b5cfff03f29_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309069d6bd64b595f4de2a59b5b5cfff03f29_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:10,565 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/815671cb5bea4ada8419c23b86a74a36, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:10,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/815671cb5bea4ada8419c23b86a74a36 is 175, key is test_row_0/A:col10/1694031309423/Put/seqid=0 2023-09-06T20:15:10,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031370566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031370566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031370566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031370567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031370568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031370677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031370678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031370678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031370678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031370678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-06T20:15:10,714 INFO [Thread-2073 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 111 completed 2023-09-06T20:15:10,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:10,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2023-09-06T20:15:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=113 2023-09-06T20:15:10,717 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:10,718 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:10,718 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:10,738 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/e61736ad7a484bc4b3df6cd9fc88f0f4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/e61736ad7a484bc4b3df6cd9fc88f0f4 2023-09-06T20:15:10,742 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/B of 3bb591e2a9ad6f7c9630808320336906 into e61736ad7a484bc4b3df6cd9fc88f0f4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:10,742 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:10,742 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/B, priority=13, startTime=1694031310290; duration=0sec 2023-09-06T20:15:10,742 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:10,742 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:B 2023-09-06T20:15:10,742 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:10,743 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:10,743 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/C is initiating minor compaction (all files) 2023-09-06T20:15:10,743 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/C in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:10,743 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/954b3a9bb66d4930ac5f3583ac717b3a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/bdbe6debf3504c34b1076b790d961106, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/0661da3230f34d3faf6047111114679a] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=35.1 K 2023-09-06T20:15:10,744 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 954b3a9bb66d4930ac5f3583ac717b3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1694031306619 2023-09-06T20:15:10,744 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting bdbe6debf3504c34b1076b790d961106, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1694031306646 2023-09-06T20:15:10,745 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 0661da3230f34d3faf6047111114679a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1694031307272 2023-09-06T20:15:10,771 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#C#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:10,772 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/96e221ae23e5401a8e159ec883dcf452 is 50, key is test_row_0/C:col10/1694031309404/Put/seqid=0 2023-09-06T20:15:10,773 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/9b2beb5435044c91b57b8ab617c5786b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/9b2beb5435044c91b57b8ab617c5786b 2023-09-06T20:15:10,782 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/A of 3bb591e2a9ad6f7c9630808320336906 into 9b2beb5435044c91b57b8ab617c5786b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:10,782 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:10,782 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/A, priority=13, startTime=1694031310290; duration=0sec 2023-09-06T20:15:10,782 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:10,782 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:A 2023-09-06T20:15:10,789 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/96e221ae23e5401a8e159ec883dcf452 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/96e221ae23e5401a8e159ec883dcf452 2023-09-06T20:15:10,800 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/C of 3bb591e2a9ad6f7c9630808320336906 into 96e221ae23e5401a8e159ec883dcf452(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:10,800 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:10,801 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/C, priority=13, startTime=1694031310290; duration=0sec 2023-09-06T20:15:10,801 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:10,801 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:C 2023-09-06T20:15:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=113 2023-09-06T20:15:10,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2023-09-06T20:15:10,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:10,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:10,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:10,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(53): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:10,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:10,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031370883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031370883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031370883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031370883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031370883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:10,970 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/815671cb5bea4ada8419c23b86a74a36 2023-09-06T20:15:10,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/13baf40becae4b31a96025838f9672f4 is 50, key is test_row_0/B:col10/1694031309423/Put/seqid=0 2023-09-06T20:15:10,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/13baf40becae4b31a96025838f9672f4 2023-09-06T20:15:10,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/5d889ee1955d4b8fa079239a928e6cb7 is 50, key is test_row_0/C:col10/1694031309423/Put/seqid=0 2023-09-06T20:15:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=113 2023-09-06T20:15:11,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2023-09-06T20:15:11,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:11,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:11,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:11,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(53): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:11,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:11,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2023-09-06T20:15:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:11,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(53): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:11,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:11,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031371188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031371188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031371189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031371189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031371190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=113 2023-09-06T20:15:11,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2023-09-06T20:15:11,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:11,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:11,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:11,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(53): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:11,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:11,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/5d889ee1955d4b8fa079239a928e6cb7 2023-09-06T20:15:11,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/815671cb5bea4ada8419c23b86a74a36 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/815671cb5bea4ada8419c23b86a74a36 2023-09-06T20:15:11,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/815671cb5bea4ada8419c23b86a74a36, entries=200, sequenceid=79, filesize=38.6 K 2023-09-06T20:15:11,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/13baf40becae4b31a96025838f9672f4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/13baf40becae4b31a96025838f9672f4 2023-09-06T20:15:11,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/13baf40becae4b31a96025838f9672f4, entries=150, sequenceid=79, filesize=11.7 K 2023-09-06T20:15:11,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/5d889ee1955d4b8fa079239a928e6cb7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/5d889ee1955d4b8fa079239a928e6cb7 2023-09-06T20:15:11,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/5d889ee1955d4b8fa079239a928e6cb7, entries=150, sequenceid=79, filesize=11.7 K 2023-09-06T20:15:11,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 3bb591e2a9ad6f7c9630808320336906 in 876ms, sequenceid=79, compaction requested=false 2023-09-06T20:15:11,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:11,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2023-09-06T20:15:11,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:11,482 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2023-09-06T20:15:11,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:11,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:11,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:11,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:11,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:11,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:11,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090614c81fc89c574d08a3c76b92ed401214_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031310566/Put/seqid=0 2023-09-06T20:15:11,514 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090614c81fc89c574d08a3c76b92ed401214_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090614c81fc89c574d08a3c76b92ed401214_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:11,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/8b7daf50efcc45d0b7b9f92e4673cdf7, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:11,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/8b7daf50efcc45d0b7b9f92e4673cdf7 is 175, key is test_row_0/A:col10/1694031310566/Put/seqid=0 2023-09-06T20:15:11,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:11,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:11,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031371712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031371713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031371713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031371716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031371717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=113 2023-09-06T20:15:11,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031371819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031371820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031371820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031371823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031371823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:11,925 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/8b7daf50efcc45d0b7b9f92e4673cdf7 2023-09-06T20:15:11,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/1bba22018fdb4541af6e448c030304c1 is 50, key is test_row_0/B:col10/1694031310566/Put/seqid=0 2023-09-06T20:15:12,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031372022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031372026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031372027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031372027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031372027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031372324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031372332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031372332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031372332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031372334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,343 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/1bba22018fdb4541af6e448c030304c1 2023-09-06T20:15:12,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/f76895f6b7584c9b920abe776e43705e is 50, key is test_row_0/C:col10/1694031310566/Put/seqid=0 2023-09-06T20:15:12,367 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/f76895f6b7584c9b920abe776e43705e 2023-09-06T20:15:12,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/8b7daf50efcc45d0b7b9f92e4673cdf7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8b7daf50efcc45d0b7b9f92e4673cdf7 2023-09-06T20:15:12,374 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8b7daf50efcc45d0b7b9f92e4673cdf7, entries=150, sequenceid=96, filesize=30.2 K 2023-09-06T20:15:12,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/1bba22018fdb4541af6e448c030304c1 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/1bba22018fdb4541af6e448c030304c1 2023-09-06T20:15:12,381 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/1bba22018fdb4541af6e448c030304c1, entries=150, sequenceid=96, filesize=11.7 K 2023-09-06T20:15:12,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/f76895f6b7584c9b920abe776e43705e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f76895f6b7584c9b920abe776e43705e 2023-09-06T20:15:12,385 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f76895f6b7584c9b920abe776e43705e, entries=150, sequenceid=96, filesize=11.7 K 2023-09-06T20:15:12,386 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3022): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3bb591e2a9ad6f7c9630808320336906 in 904ms, sequenceid=96, compaction requested=true 2023-09-06T20:15:12,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:12,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:12,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=114 2023-09-06T20:15:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=114 2023-09-06T20:15:12,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=114, resume processing ppid=113 2023-09-06T20:15:12,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=114, ppid=113, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6690 sec 2023-09-06T20:15:12,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=113, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.6740 sec 2023-09-06T20:15:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=113 2023-09-06T20:15:12,822 INFO [Thread-2073 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 113 completed 2023-09-06T20:15:12,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:12,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2023-09-06T20:15:12,826 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:12,826 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:12,827 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-06T20:15:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:12,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-06T20:15:12,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:12,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:12,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:12,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:12,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:12,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:12,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906f8b3fb7b02b34af2a98ce76618717074_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031311715/Put/seqid=0 2023-09-06T20:15:12,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031372850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031372855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031372856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031372858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031372857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-06T20:15:12,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031372960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031372962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031372962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031372966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:12,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031372966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:12,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2023-09-06T20:15:12,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:12,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:12,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:12,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(53): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:12,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:12,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:13,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-06T20:15:13,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2023-09-06T20:15:13,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:13,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:13,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:13,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(53): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:13,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:13,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:13,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:13,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031373168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:13,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:13,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031373169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:13,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:13,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031373172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:13,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:13,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031373174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:13,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:13,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031373175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:13,259 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906f8b3fb7b02b34af2a98ce76618717074_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f8b3fb7b02b34af2a98ce76618717074_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:13,260 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/e89263ec8daf48cca8b4108390e59974, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:13,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/e89263ec8daf48cca8b4108390e59974 is 175, key is test_row_0/A:col10/1694031311715/Put/seqid=0 2023-09-06T20:15:13,267 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/e89263ec8daf48cca8b4108390e59974 2023-09-06T20:15:13,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/b78832007f944cf5808c5793ab3d3fb1 is 50, key is test_row_0/B:col10/1694031311715/Put/seqid=0 2023-09-06T20:15:13,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2023-09-06T20:15:13,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:13,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:13,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:13,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(53): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:13,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-06T20:15:13,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2023-09-06T20:15:13,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:13,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:13,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:13,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(53): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:13,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:13,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031373472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:13,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031373478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:13,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031373478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:13,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:13,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031373479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031373479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:13,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2023-09-06T20:15:13,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:13,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:13,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:13,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(53): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:13,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:13,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/b78832007f944cf5808c5793ab3d3fb1 2023-09-06T20:15:13,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/f307a165cba44528b2c022ba7a3fb467 is 50, key is test_row_0/C:col10/1694031311715/Put/seqid=0 2023-09-06T20:15:13,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/f307a165cba44528b2c022ba7a3fb467 2023-09-06T20:15:13,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/e89263ec8daf48cca8b4108390e59974 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/e89263ec8daf48cca8b4108390e59974 2023-09-06T20:15:13,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/e89263ec8daf48cca8b4108390e59974, entries=150, sequenceid=119, filesize=30.2 K 2023-09-06T20:15:13,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/b78832007f944cf5808c5793ab3d3fb1 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b78832007f944cf5808c5793ab3d3fb1 2023-09-06T20:15:13,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b78832007f944cf5808c5793ab3d3fb1, entries=150, sequenceid=119, filesize=11.7 K 2023-09-06T20:15:13,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/f307a165cba44528b2c022ba7a3fb467 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f307a165cba44528b2c022ba7a3fb467 2023-09-06T20:15:13,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f307a165cba44528b2c022ba7a3fb467, entries=150, sequenceid=119, filesize=11.7 K 2023-09-06T20:15:13,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 3bb591e2a9ad6f7c9630808320336906 in 872ms, sequenceid=119, compaction requested=true 2023-09-06T20:15:13,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:13,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:13,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:13,710 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:13,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:13,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:13,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:13,710 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:13,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:13,712 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132449 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:13,712 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/A is initiating minor compaction (all files) 2023-09-06T20:15:13,712 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/A in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:13,712 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/9b2beb5435044c91b57b8ab617c5786b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/815671cb5bea4ada8419c23b86a74a36, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8b7daf50efcc45d0b7b9f92e4673cdf7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/e89263ec8daf48cca8b4108390e59974] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=129.3 K 2023-09-06T20:15:13,712 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:13,712 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/9b2beb5435044c91b57b8ab617c5786b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/815671cb5bea4ada8419c23b86a74a36, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8b7daf50efcc45d0b7b9f92e4673cdf7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/e89263ec8daf48cca8b4108390e59974] 2023-09-06T20:15:13,713 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9b2beb5435044c91b57b8ab617c5786b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1694031307272 2023-09-06T20:15:13,713 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 815671cb5bea4ada8419c23b86a74a36, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1694031309423 2023-09-06T20:15:13,713 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48039 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:13,713 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/B is initiating minor compaction (all files) 2023-09-06T20:15:13,713 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/B in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:13,713 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/e61736ad7a484bc4b3df6cd9fc88f0f4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/13baf40becae4b31a96025838f9672f4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/1bba22018fdb4541af6e448c030304c1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b78832007f944cf5808c5793ab3d3fb1] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=46.9 K 2023-09-06T20:15:13,714 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8b7daf50efcc45d0b7b9f92e4673cdf7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1694031310565 2023-09-06T20:15:13,714 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting e61736ad7a484bc4b3df6cd9fc88f0f4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1694031307272 2023-09-06T20:15:13,714 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting e89263ec8daf48cca8b4108390e59974, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1694031311715 2023-09-06T20:15:13,715 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 13baf40becae4b31a96025838f9672f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1694031309423 2023-09-06T20:15:13,715 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 1bba22018fdb4541af6e448c030304c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1694031310565 2023-09-06T20:15:13,722 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b78832007f944cf5808c5793ab3d3fb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1694031311715 2023-09-06T20:15:13,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2023-09-06T20:15:13,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:13,748 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-06T20:15:13,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:13,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:13,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:13,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:13,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:13,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:13,750 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:13,755 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#B#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:13,756 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/32ed6358ebca4f5192b8f15e46e36888 is 50, key is test_row_0/B:col10/1694031311715/Put/seqid=0 2023-09-06T20:15:13,762 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309067f17c6141b6a4a438605fff56477656a_3bb591e2a9ad6f7c9630808320336906 store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:13,764 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309067f17c6141b6a4a438605fff56477656a_3bb591e2a9ad6f7c9630808320336906, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:13,764 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309067f17c6141b6a4a438605fff56477656a_3bb591e2a9ad6f7c9630808320336906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:13,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906ae89659a58d64ab6aa58146cff8c5acf_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031312849/Put/seqid=0 2023-09-06T20:15:13,816 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/32ed6358ebca4f5192b8f15e46e36888 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/32ed6358ebca4f5192b8f15e46e36888 2023-09-06T20:15:13,821 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/B of 3bb591e2a9ad6f7c9630808320336906 into 32ed6358ebca4f5192b8f15e46e36888(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:13,821 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:13,821 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/B, priority=12, startTime=1694031313710; duration=0sec 2023-09-06T20:15:13,821 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:13,821 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:B 2023-09-06T20:15:13,821 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:13,824 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48039 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:13,824 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/C is initiating minor compaction (all files) 2023-09-06T20:15:13,824 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/C in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:13,824 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/96e221ae23e5401a8e159ec883dcf452, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/5d889ee1955d4b8fa079239a928e6cb7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f76895f6b7584c9b920abe776e43705e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f307a165cba44528b2c022ba7a3fb467] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=46.9 K 2023-09-06T20:15:13,825 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 96e221ae23e5401a8e159ec883dcf452, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1694031307272 2023-09-06T20:15:13,826 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 5d889ee1955d4b8fa079239a928e6cb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1694031309423 2023-09-06T20:15:13,827 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting f76895f6b7584c9b920abe776e43705e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1694031310565 2023-09-06T20:15:13,827 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting f307a165cba44528b2c022ba7a3fb467, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1694031311715 2023-09-06T20:15:13,839 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#C#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:13,839 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/ca05207bf2b446c6bb45a4f99c07abcf is 50, key is test_row_0/C:col10/1694031311715/Put/seqid=0 2023-09-06T20:15:13,849 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/ca05207bf2b446c6bb45a4f99c07abcf as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/ca05207bf2b446c6bb45a4f99c07abcf 2023-09-06T20:15:13,855 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/C of 3bb591e2a9ad6f7c9630808320336906 into ca05207bf2b446c6bb45a4f99c07abcf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:13,855 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:13,855 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/C, priority=12, startTime=1694031313710; duration=0sec 2023-09-06T20:15:13,855 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:13,855 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:C 2023-09-06T20:15:13,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-06T20:15:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:13,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:14,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031374009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031374010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031374011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031374013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031374014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031374117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031374117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031374117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031374118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031374120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,204 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#A#compaction#358 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:14,205 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/8bf0763d6aea4d338890e3854a248344 is 175, key is test_row_0/A:col10/1694031311715/Put/seqid=0 2023-09-06T20:15:14,230 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906ae89659a58d64ab6aa58146cff8c5acf_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906ae89659a58d64ab6aa58146cff8c5acf_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:14,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/8ff24b28822349a6b28d18327392b571, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:14,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/8ff24b28822349a6b28d18327392b571 is 175, key is test_row_0/A:col10/1694031312849/Put/seqid=0 2023-09-06T20:15:14,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031374320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031374320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031374320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031374320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031374327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,624 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/8bf0763d6aea4d338890e3854a248344 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8bf0763d6aea4d338890e3854a248344 2023-09-06T20:15:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031374623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031374624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031374624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031374625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,629 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/A of 3bb591e2a9ad6f7c9630808320336906 into 8bf0763d6aea4d338890e3854a248344(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:14,629 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:14,629 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/A, priority=12, startTime=1694031313710; duration=0sec 2023-09-06T20:15:14,629 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:14,629 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:A 2023-09-06T20:15:14,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031374639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:14,646 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/8ff24b28822349a6b28d18327392b571 2023-09-06T20:15:14,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/be96018f92f64e449536e62f8394053e is 50, key is test_row_0/B:col10/1694031312849/Put/seqid=0 2023-09-06T20:15:14,657 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/be96018f92f64e449536e62f8394053e 2023-09-06T20:15:14,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/fde6cc593541404580d3ffb371b1b235 is 50, key is test_row_0/C:col10/1694031312849/Put/seqid=0 2023-09-06T20:15:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-06T20:15:15,092 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/fde6cc593541404580d3ffb371b1b235 2023-09-06T20:15:15,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/8ff24b28822349a6b28d18327392b571 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8ff24b28822349a6b28d18327392b571 2023-09-06T20:15:15,100 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8ff24b28822349a6b28d18327392b571, entries=150, sequenceid=132, filesize=30.3 K 2023-09-06T20:15:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/be96018f92f64e449536e62f8394053e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/be96018f92f64e449536e62f8394053e 2023-09-06T20:15:15,105 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/be96018f92f64e449536e62f8394053e, entries=150, sequenceid=132, filesize=11.8 K 2023-09-06T20:15:15,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/fde6cc593541404580d3ffb371b1b235 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/fde6cc593541404580d3ffb371b1b235 2023-09-06T20:15:15,109 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/fde6cc593541404580d3ffb371b1b235, entries=150, sequenceid=132, filesize=11.8 K 2023-09-06T20:15:15,110 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3bb591e2a9ad6f7c9630808320336906 in 1362ms, sequenceid=132, compaction requested=false 2023-09-06T20:15:15,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:15,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:15,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=116 2023-09-06T20:15:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=116 2023-09-06T20:15:15,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=116, resume processing ppid=115 2023-09-06T20:15:15,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=116, ppid=115, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2850 sec 2023-09-06T20:15:15,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=115, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.2900 sec 2023-09-06T20:15:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:15,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-06T20:15:15,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:15,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:15,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:15,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:15,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:15,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:15,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906f8173b701c6d4017b64899205906ac19_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031314012/Put/seqid=0 2023-09-06T20:15:15,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031375141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031375145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,150 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906f8173b701c6d4017b64899205906ac19_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f8173b701c6d4017b64899205906ac19_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:15,150 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/5a81c87b04304990ab20a1331c37b756, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:15,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/5a81c87b04304990ab20a1331c37b756 is 175, key is test_row_0/A:col10/1694031314012/Put/seqid=0 2023-09-06T20:15:15,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031375146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031375147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031375148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/5a81c87b04304990ab20a1331c37b756 2023-09-06T20:15:15,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/04b1aba045c44717834b219b9931b8b9 is 50, key is test_row_0/B:col10/1694031314012/Put/seqid=0 2023-09-06T20:15:15,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031375248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031375249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031375255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031375255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031375452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031375452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031375459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031375461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/04b1aba045c44717834b219b9931b8b9 2023-09-06T20:15:15,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/86bc3c80c06b487ab1055f0a9fa33f45 is 50, key is test_row_0/C:col10/1694031314012/Put/seqid=0 2023-09-06T20:15:15,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031375758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031375758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031375766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031375768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:15,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/86bc3c80c06b487ab1055f0a9fa33f45 2023-09-06T20:15:15,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/5a81c87b04304990ab20a1331c37b756 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/5a81c87b04304990ab20a1331c37b756 2023-09-06T20:15:15,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/5a81c87b04304990ab20a1331c37b756, entries=200, sequenceid=159, filesize=38.8 K 2023-09-06T20:15:16,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/04b1aba045c44717834b219b9931b8b9 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/04b1aba045c44717834b219b9931b8b9 2023-09-06T20:15:16,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/04b1aba045c44717834b219b9931b8b9, entries=150, sequenceid=159, filesize=11.8 K 2023-09-06T20:15:16,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/86bc3c80c06b487ab1055f0a9fa33f45 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/86bc3c80c06b487ab1055f0a9fa33f45 2023-09-06T20:15:16,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/86bc3c80c06b487ab1055f0a9fa33f45, entries=150, sequenceid=159, filesize=11.8 K 2023-09-06T20:15:16,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3bb591e2a9ad6f7c9630808320336906 in 876ms, sequenceid=159, compaction requested=true 2023-09-06T20:15:16,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:16,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:16,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:16,008 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:16,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:16,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:16,009 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:16,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:16,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:16,010 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101948 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:16,010 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36442 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:16,010 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/A is initiating minor compaction (all files) 2023-09-06T20:15:16,010 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/B is initiating minor compaction (all files) 2023-09-06T20:15:16,010 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/B in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:16,010 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/A in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:16,010 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8bf0763d6aea4d338890e3854a248344, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8ff24b28822349a6b28d18327392b571, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/5a81c87b04304990ab20a1331c37b756] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=99.6 K 2023-09-06T20:15:16,010 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/32ed6358ebca4f5192b8f15e46e36888, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/be96018f92f64e449536e62f8394053e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/04b1aba045c44717834b219b9931b8b9] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=35.6 K 2023-09-06T20:15:16,010 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:16,010 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8bf0763d6aea4d338890e3854a248344, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8ff24b28822349a6b28d18327392b571, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/5a81c87b04304990ab20a1331c37b756] 2023-09-06T20:15:16,010 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 32ed6358ebca4f5192b8f15e46e36888, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1694031311715 2023-09-06T20:15:16,011 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8bf0763d6aea4d338890e3854a248344, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1694031311715 2023-09-06T20:15:16,011 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting be96018f92f64e449536e62f8394053e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1694031312849 2023-09-06T20:15:16,011 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8ff24b28822349a6b28d18327392b571, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1694031312849 2023-09-06T20:15:16,012 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 04b1aba045c44717834b219b9931b8b9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1694031314010 2023-09-06T20:15:16,012 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5a81c87b04304990ab20a1331c37b756, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1694031314001 2023-09-06T20:15:16,027 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#B#compaction#367 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:16,027 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/843253acd4594764b5f06a0b80de2fa7 is 50, key is test_row_0/B:col10/1694031314012/Put/seqid=0 2023-09-06T20:15:16,030 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:16,046 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230906bba1a0540f234627992813be315e5e7a_3bb591e2a9ad6f7c9630808320336906 store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:16,047 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230906bba1a0540f234627992813be315e5e7a_3bb591e2a9ad6f7c9630808320336906, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:16,047 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906bba1a0540f234627992813be315e5e7a_3bb591e2a9ad6f7c9630808320336906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:16,053 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/843253acd4594764b5f06a0b80de2fa7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/843253acd4594764b5f06a0b80de2fa7 2023-09-06T20:15:16,057 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/B of 3bb591e2a9ad6f7c9630808320336906 into 843253acd4594764b5f06a0b80de2fa7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:16,057 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:16,057 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/B, priority=13, startTime=1694031316008; duration=0sec 2023-09-06T20:15:16,057 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:16,058 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:B 2023-09-06T20:15:16,058 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:16,059 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36442 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:16,059 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/C is initiating minor compaction (all files) 2023-09-06T20:15:16,059 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/C in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:16,059 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/ca05207bf2b446c6bb45a4f99c07abcf, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/fde6cc593541404580d3ffb371b1b235, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/86bc3c80c06b487ab1055f0a9fa33f45] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=35.6 K 2023-09-06T20:15:16,059 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting ca05207bf2b446c6bb45a4f99c07abcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1694031311715 2023-09-06T20:15:16,060 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting fde6cc593541404580d3ffb371b1b235, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1694031312849 2023-09-06T20:15:16,060 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 86bc3c80c06b487ab1055f0a9fa33f45, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1694031314010 2023-09-06T20:15:16,061 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#A#compaction#368 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:16,061 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/cc2bf3a7adfe4d76a0ac7106df8d095a is 175, key is test_row_0/A:col10/1694031314012/Put/seqid=0 2023-09-06T20:15:16,067 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#C#compaction#369 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:16,068 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/ce0dd2e096d54cfb925c4b06e0da9967 is 50, key is test_row_0/C:col10/1694031314012/Put/seqid=0 2023-09-06T20:15:16,073 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/cc2bf3a7adfe4d76a0ac7106df8d095a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/cc2bf3a7adfe4d76a0ac7106df8d095a 2023-09-06T20:15:16,077 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/A of 3bb591e2a9ad6f7c9630808320336906 into cc2bf3a7adfe4d76a0ac7106df8d095a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:16,077 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:16,077 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/A, priority=13, startTime=1694031316008; duration=0sec 2023-09-06T20:15:16,077 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:16,077 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:A 2023-09-06T20:15:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:16,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-06T20:15:16,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:16,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:16,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:16,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:16,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:16,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:16,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c7e606cd833f4f08baa585d43be6df8d_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031315144/Put/seqid=0 2023-09-06T20:15:16,180 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c7e606cd833f4f08baa585d43be6df8d_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c7e606cd833f4f08baa585d43be6df8d_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:16,181 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/eb670e40f0484a2b8d5c2d52b58b05bd, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:16,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/eb670e40f0484a2b8d5c2d52b58b05bd is 175, key is test_row_0/A:col10/1694031315144/Put/seqid=0 2023-09-06T20:15:16,187 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/eb670e40f0484a2b8d5c2d52b58b05bd 2023-09-06T20:15:16,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/0c89b0fd5ccf452f85eb38c4e521e5f9 is 50, key is test_row_0/B:col10/1694031315144/Put/seqid=0 2023-09-06T20:15:16,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/0c89b0fd5ccf452f85eb38c4e521e5f9 2023-09-06T20:15:16,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/0effa522f53940d38f6d69f2bc1ff80c is 50, key is test_row_0/C:col10/1694031315144/Put/seqid=0 2023-09-06T20:15:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031376265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031376267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:16,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031376269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:16,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031376272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:16,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031376275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:16,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031376375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:16,485 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/ce0dd2e096d54cfb925c4b06e0da9967 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/ce0dd2e096d54cfb925c4b06e0da9967 2023-09-06T20:15:16,489 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/C of 3bb591e2a9ad6f7c9630808320336906 into ce0dd2e096d54cfb925c4b06e0da9967(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:16,489 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:16,489 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/C, priority=13, startTime=1694031316009; duration=0sec 2023-09-06T20:15:16,489 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:16,489 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:C 2023-09-06T20:15:16,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031376580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:16,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/0effa522f53940d38f6d69f2bc1ff80c 2023-09-06T20:15:16,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/eb670e40f0484a2b8d5c2d52b58b05bd as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/eb670e40f0484a2b8d5c2d52b58b05bd 2023-09-06T20:15:16,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/eb670e40f0484a2b8d5c2d52b58b05bd, entries=200, sequenceid=174, filesize=38.8 K 2023-09-06T20:15:16,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/0c89b0fd5ccf452f85eb38c4e521e5f9 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/0c89b0fd5ccf452f85eb38c4e521e5f9 2023-09-06T20:15:16,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/0c89b0fd5ccf452f85eb38c4e521e5f9, entries=150, sequenceid=174, filesize=11.8 K 2023-09-06T20:15:16,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/0effa522f53940d38f6d69f2bc1ff80c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/0effa522f53940d38f6d69f2bc1ff80c 2023-09-06T20:15:16,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/0effa522f53940d38f6d69f2bc1ff80c, entries=150, sequenceid=174, filesize=11.8 K 2023-09-06T20:15:16,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3bb591e2a9ad6f7c9630808320336906 in 480ms, sequenceid=174, compaction requested=false 2023-09-06T20:15:16,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:16,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-06T20:15:16,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:16,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:16,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:16,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:16,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:16,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:16,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906eeb304e70e084ab2bf15ed3c4027493f_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031316268/Put/seqid=0 2023-09-06T20:15:16,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031376924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-06T20:15:16,934 INFO [Thread-2073 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 115 completed 2023-09-06T20:15:16,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2023-09-06T20:15:16,937 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-06T20:15:16,937 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:16,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:17,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031377029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-06T20:15:17,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-06T20:15:17,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:17,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:17,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:17,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-06T20:15:17,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031377236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:17,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-06T20:15:17,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:17,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:17,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:17,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031377276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:17,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031377279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:17,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031377282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:17,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031377284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:17,304 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906eeb304e70e084ab2bf15ed3c4027493f_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906eeb304e70e084ab2bf15ed3c4027493f_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:17,305 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/1c3856a484744133925817978581b964, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:17,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/1c3856a484744133925817978581b964 is 175, key is test_row_0/A:col10/1694031316268/Put/seqid=0 2023-09-06T20:15:17,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-06T20:15:17,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:17,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:17,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:17,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-06T20:15:17,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:17,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031377542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:17,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-06T20:15:17,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:17,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:17,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:17,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-06T20:15:17,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:17,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:17,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:17,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,712 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/1c3856a484744133925817978581b964 2023-09-06T20:15:17,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/b8e8a335deaf481fb2be058efab45de7 is 50, key is test_row_0/B:col10/1694031316268/Put/seqid=0 2023-09-06T20:15:17,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-06T20:15:17,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:17,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:17,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:17,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:18,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-06T20:15:18,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:18,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:18,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:18,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:18,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-06T20:15:18,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031378052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:18,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/b8e8a335deaf481fb2be058efab45de7 2023-09-06T20:15:18,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/2d8656c95d174d4986962d26bccf4e92 is 50, key is test_row_0/C:col10/1694031316268/Put/seqid=0 2023-09-06T20:15:18,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-06T20:15:18,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:18,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:18,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:18,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:18,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:18,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-06T20:15:18,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:18,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:18,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:18,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:18,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:18,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-06T20:15:18,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:18,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:18,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:18,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:18,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:18,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/2d8656c95d174d4986962d26bccf4e92 2023-09-06T20:15:18,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/1c3856a484744133925817978581b964 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/1c3856a484744133925817978581b964 2023-09-06T20:15:18,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/1c3856a484744133925817978581b964, entries=200, sequenceid=199, filesize=38.8 K 2023-09-06T20:15:18,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/b8e8a335deaf481fb2be058efab45de7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b8e8a335deaf481fb2be058efab45de7 2023-09-06T20:15:18,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b8e8a335deaf481fb2be058efab45de7, entries=150, sequenceid=199, filesize=11.8 K 2023-09-06T20:15:18,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/2d8656c95d174d4986962d26bccf4e92 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/2d8656c95d174d4986962d26bccf4e92 2023-09-06T20:15:18,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/2d8656c95d174d4986962d26bccf4e92, entries=150, sequenceid=199, filesize=11.8 K 2023-09-06T20:15:18,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3bb591e2a9ad6f7c9630808320336906 in 1671ms, sequenceid=199, compaction requested=true 2023-09-06T20:15:18,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:18,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:18,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:18,559 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:18,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:18,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:18,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:18,559 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:18,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:18,560 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:18,560 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110894 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:18,560 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/B is initiating minor compaction (all files) 2023-09-06T20:15:18,560 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/A is initiating minor compaction (all files) 2023-09-06T20:15:18,560 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/B in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:18,560 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/A in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:18,560 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/cc2bf3a7adfe4d76a0ac7106df8d095a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/eb670e40f0484a2b8d5c2d52b58b05bd, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/1c3856a484744133925817978581b964] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=108.3 K 2023-09-06T20:15:18,560 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/843253acd4594764b5f06a0b80de2fa7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/0c89b0fd5ccf452f85eb38c4e521e5f9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b8e8a335deaf481fb2be058efab45de7] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=35.9 K 2023-09-06T20:15:18,560 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:18,560 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/cc2bf3a7adfe4d76a0ac7106df8d095a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/eb670e40f0484a2b8d5c2d52b58b05bd, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/1c3856a484744133925817978581b964] 2023-09-06T20:15:18,563 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 843253acd4594764b5f06a0b80de2fa7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1694031314010 2023-09-06T20:15:18,563 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 0c89b0fd5ccf452f85eb38c4e521e5f9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1694031315144 2023-09-06T20:15:18,563 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b8e8a335deaf481fb2be058efab45de7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1694031316260 2023-09-06T20:15:18,569 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#B#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:18,570 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting cc2bf3a7adfe4d76a0ac7106df8d095a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1694031314010 2023-09-06T20:15:18,570 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/dbe0b448b98d4162a9e95e99cf0180f5 is 50, key is test_row_0/B:col10/1694031316268/Put/seqid=0 2023-09-06T20:15:18,570 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting eb670e40f0484a2b8d5c2d52b58b05bd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1694031315143 2023-09-06T20:15:18,571 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1c3856a484744133925817978581b964, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1694031316245 2023-09-06T20:15:18,577 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:18,582 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230906f4c7666763d242ff99cd443ff8a3749b_3bb591e2a9ad6f7c9630808320336906 store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:18,583 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230906f4c7666763d242ff99cd443ff8a3749b_3bb591e2a9ad6f7c9630808320336906, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:18,583 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906f4c7666763d242ff99cd443ff8a3749b_3bb591e2a9ad6f7c9630808320336906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:18,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-06T20:15:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:18,617 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:15:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:18,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906f19fb2c5914e4c61b9d1fed4676b603a_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031316922/Put/seqid=0 2023-09-06T20:15:18,985 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/dbe0b448b98d4162a9e95e99cf0180f5 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/dbe0b448b98d4162a9e95e99cf0180f5 2023-09-06T20:15:18,989 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/B of 3bb591e2a9ad6f7c9630808320336906 into dbe0b448b98d4162a9e95e99cf0180f5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:18,989 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:18,989 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/B, priority=13, startTime=1694031318559; duration=0sec 2023-09-06T20:15:18,989 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:18,989 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:B 2023-09-06T20:15:18,989 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:18,990 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:18,990 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/C is initiating minor compaction (all files) 2023-09-06T20:15:18,990 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/C in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:18,990 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#A#compaction#377 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:18,990 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/ce0dd2e096d54cfb925c4b06e0da9967, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/0effa522f53940d38f6d69f2bc1ff80c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/2d8656c95d174d4986962d26bccf4e92] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=35.9 K 2023-09-06T20:15:18,991 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting ce0dd2e096d54cfb925c4b06e0da9967, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1694031314010 2023-09-06T20:15:18,991 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/3ffa6678e292471c906e8becef4e3de5 is 175, key is test_row_0/A:col10/1694031316268/Put/seqid=0 2023-09-06T20:15:18,991 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 0effa522f53940d38f6d69f2bc1ff80c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1694031315144 2023-09-06T20:15:18,991 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 2d8656c95d174d4986962d26bccf4e92, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1694031316260 2023-09-06T20:15:19,003 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#C#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:19,003 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/31965c52fb47460281c770b759141047 is 50, key is test_row_0/C:col10/1694031316268/Put/seqid=0 2023-09-06T20:15:19,005 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/3ffa6678e292471c906e8becef4e3de5 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/3ffa6678e292471c906e8becef4e3de5 2023-09-06T20:15:19,010 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/A of 3bb591e2a9ad6f7c9630808320336906 into 3ffa6678e292471c906e8becef4e3de5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:19,010 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:19,010 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/A, priority=13, startTime=1694031318559; duration=0sec 2023-09-06T20:15:19,010 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:19,010 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:A 2023-09-06T20:15:19,018 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/31965c52fb47460281c770b759141047 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/31965c52fb47460281c770b759141047 2023-09-06T20:15:19,023 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/C of 3bb591e2a9ad6f7c9630808320336906 into 31965c52fb47460281c770b759141047(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:19,023 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:19,023 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/C, priority=13, startTime=1694031318559; duration=0sec 2023-09-06T20:15:19,024 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:19,024 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:C 2023-09-06T20:15:19,040 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906f19fb2c5914e4c61b9d1fed4676b603a_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f19fb2c5914e4c61b9d1fed4676b603a_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-06T20:15:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/697c9bdaa023447ebff62e3262d24e5d, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:19,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/697c9bdaa023447ebff62e3262d24e5d is 175, key is test_row_0/A:col10/1694031316922/Put/seqid=0 2023-09-06T20:15:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:19,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:19,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031379166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:19,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031379270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:19,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031379292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:19,297 DEBUG [Thread-2065 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4148 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., hostname=jenkins-hbase3.apache.org,33275,1694031217268, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:130) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-06T20:15:19,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031379292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:19,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031379294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:19,299 DEBUG [Thread-2063 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4154 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., hostname=jenkins-hbase3.apache.org,33275,1694031217268, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:130) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-06T20:15:19,300 DEBUG [Thread-2067 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4158 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., hostname=jenkins-hbase3.apache.org,33275,1694031217268, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:130) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-06T20:15:19,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031379297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:19,302 DEBUG [Thread-2071 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4155 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., hostname=jenkins-hbase3.apache.org,33275,1694031217268, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:130) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-06T20:15:19,453 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/697c9bdaa023447ebff62e3262d24e5d 2023-09-06T20:15:19,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/1ee75fd7993343359846142f11abd2d2 is 50, key is test_row_0/B:col10/1694031316922/Put/seqid=0 2023-09-06T20:15:19,464 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/1ee75fd7993343359846142f11abd2d2 2023-09-06T20:15:19,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/3b7fc437769446ccaa53e99cfaa1b268 is 50, key is test_row_0/C:col10/1694031316922/Put/seqid=0 2023-09-06T20:15:19,476 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/3b7fc437769446ccaa53e99cfaa1b268 2023-09-06T20:15:19,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031379476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:19,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/697c9bdaa023447ebff62e3262d24e5d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/697c9bdaa023447ebff62e3262d24e5d 2023-09-06T20:15:19,485 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/697c9bdaa023447ebff62e3262d24e5d, entries=150, sequenceid=211, filesize=30.4 K 2023-09-06T20:15:19,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/1ee75fd7993343359846142f11abd2d2 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/1ee75fd7993343359846142f11abd2d2 2023-09-06T20:15:19,489 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/1ee75fd7993343359846142f11abd2d2, entries=150, sequenceid=211, filesize=11.8 K 2023-09-06T20:15:19,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/3b7fc437769446ccaa53e99cfaa1b268 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/3b7fc437769446ccaa53e99cfaa1b268 2023-09-06T20:15:19,496 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/3b7fc437769446ccaa53e99cfaa1b268, entries=150, sequenceid=211, filesize=11.8 K 2023-09-06T20:15:19,497 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3bb591e2a9ad6f7c9630808320336906 in 880ms, sequenceid=211, compaction requested=false 2023-09-06T20:15:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=118 2023-09-06T20:15:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=118 2023-09-06T20:15:19,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=118, resume processing ppid=117 2023-09-06T20:15:19,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=118, ppid=117, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5610 sec 2023-09-06T20:15:19,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=117, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.5660 sec 2023-09-06T20:15:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:19,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-06T20:15:19,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:19,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:19,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:19,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:19,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:19,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:19,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c5c89ba9b13a4a2aad6a8550d84362df_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031319784/Put/seqid=0 2023-09-06T20:15:19,807 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c5c89ba9b13a4a2aad6a8550d84362df_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c5c89ba9b13a4a2aad6a8550d84362df_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:19,808 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/dc1ff4d87fd043a7af1cd2dc9629b17b, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:19,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/dc1ff4d87fd043a7af1cd2dc9629b17b is 175, key is test_row_0/A:col10/1694031319784/Put/seqid=0 2023-09-06T20:15:19,813 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/dc1ff4d87fd043a7af1cd2dc9629b17b 2023-09-06T20:15:19,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/2c2ea285254241c3b51f0f6018313eb8 is 50, key is test_row_0/B:col10/1694031319784/Put/seqid=0 2023-09-06T20:15:19,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/2c2ea285254241c3b51f0f6018313eb8 2023-09-06T20:15:19,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:19,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/1abe0d35b90e4333a825bb96aa8815ef is 50, key is test_row_0/C:col10/1694031319784/Put/seqid=0 2023-09-06T20:15:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031379830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:19,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031379939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:20,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031380145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:20,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/1abe0d35b90e4333a825bb96aa8815ef 2023-09-06T20:15:20,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/dc1ff4d87fd043a7af1cd2dc9629b17b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/dc1ff4d87fd043a7af1cd2dc9629b17b 2023-09-06T20:15:20,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/dc1ff4d87fd043a7af1cd2dc9629b17b, entries=200, sequenceid=239, filesize=38.8 K 2023-09-06T20:15:20,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/2c2ea285254241c3b51f0f6018313eb8 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/2c2ea285254241c3b51f0f6018313eb8 2023-09-06T20:15:20,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/2c2ea285254241c3b51f0f6018313eb8, entries=150, sequenceid=239, filesize=11.8 K 2023-09-06T20:15:20,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/1abe0d35b90e4333a825bb96aa8815ef as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/1abe0d35b90e4333a825bb96aa8815ef 2023-09-06T20:15:20,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/1abe0d35b90e4333a825bb96aa8815ef, entries=150, sequenceid=239, filesize=11.8 K 2023-09-06T20:15:20,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3bb591e2a9ad6f7c9630808320336906 in 481ms, sequenceid=239, compaction requested=true 2023-09-06T20:15:20,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:20,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:20,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:20,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:20,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:20,267 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:20,267 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:20,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:20,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:20,268 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102352 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:20,268 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/A is initiating minor compaction (all files) 2023-09-06T20:15:20,268 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/A in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:20,268 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/3ffa6678e292471c906e8becef4e3de5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/697c9bdaa023447ebff62e3262d24e5d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/dc1ff4d87fd043a7af1cd2dc9629b17b] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=100.0 K 2023-09-06T20:15:20,268 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:20,269 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/B is initiating minor compaction (all files) 2023-09-06T20:15:20,269 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/B in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:20,269 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/dbe0b448b98d4162a9e95e99cf0180f5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/1ee75fd7993343359846142f11abd2d2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/2c2ea285254241c3b51f0f6018313eb8] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=36.0 K 2023-09-06T20:15:20,268 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:20,269 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/3ffa6678e292471c906e8becef4e3de5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/697c9bdaa023447ebff62e3262d24e5d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/dc1ff4d87fd043a7af1cd2dc9629b17b] 2023-09-06T20:15:20,269 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting dbe0b448b98d4162a9e95e99cf0180f5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1694031316260 2023-09-06T20:15:20,269 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3ffa6678e292471c906e8becef4e3de5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1694031316260 2023-09-06T20:15:20,270 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 1ee75fd7993343359846142f11abd2d2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1694031316903 2023-09-06T20:15:20,270 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 697c9bdaa023447ebff62e3262d24e5d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1694031316903 2023-09-06T20:15:20,270 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 2c2ea285254241c3b51f0f6018313eb8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1694031319161 2023-09-06T20:15:20,270 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting dc1ff4d87fd043a7af1cd2dc9629b17b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1694031319156 2023-09-06T20:15:20,279 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#B#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:20,280 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/cf82e62b11a0457693f1ac56cbefe42a is 50, key is test_row_0/B:col10/1694031319784/Put/seqid=0 2023-09-06T20:15:20,281 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:20,283 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2023090677171921d5244934a27953dee25f0330_3bb591e2a9ad6f7c9630808320336906 store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:20,284 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2023090677171921d5244934a27953dee25f0330_3bb591e2a9ad6f7c9630808320336906, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:20,284 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090677171921d5244934a27953dee25f0330_3bb591e2a9ad6f7c9630808320336906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:20,292 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/cf82e62b11a0457693f1ac56cbefe42a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/cf82e62b11a0457693f1ac56cbefe42a 2023-09-06T20:15:20,292 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#A#compaction#386 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:20,293 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/5623da743d5b417781fa64d34b31e113 is 175, key is test_row_0/A:col10/1694031319784/Put/seqid=0 2023-09-06T20:15:20,297 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/B of 3bb591e2a9ad6f7c9630808320336906 into cf82e62b11a0457693f1ac56cbefe42a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:20,297 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:20,297 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/B, priority=13, startTime=1694031320267; duration=0sec 2023-09-06T20:15:20,297 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:20,297 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:B 2023-09-06T20:15:20,297 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:20,299 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:20,299 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/C is initiating minor compaction (all files) 2023-09-06T20:15:20,299 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/C in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:20,299 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/31965c52fb47460281c770b759141047, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/3b7fc437769446ccaa53e99cfaa1b268, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/1abe0d35b90e4333a825bb96aa8815ef] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=36.0 K 2023-09-06T20:15:20,302 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 31965c52fb47460281c770b759141047, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1694031316260 2023-09-06T20:15:20,302 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 3b7fc437769446ccaa53e99cfaa1b268, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1694031316903 2023-09-06T20:15:20,303 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 1abe0d35b90e4333a825bb96aa8815ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1694031319161 2023-09-06T20:15:20,308 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/5623da743d5b417781fa64d34b31e113 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/5623da743d5b417781fa64d34b31e113 2023-09-06T20:15:20,309 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#C#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:20,310 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/b38e705aca114e54bf989e9f3a3b4832 is 50, key is test_row_0/C:col10/1694031319784/Put/seqid=0 2023-09-06T20:15:20,314 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/A of 3bb591e2a9ad6f7c9630808320336906 into 5623da743d5b417781fa64d34b31e113(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:20,314 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:20,314 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/A, priority=13, startTime=1694031320267; duration=0sec 2023-09-06T20:15:20,314 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:20,314 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:A 2023-09-06T20:15:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:20,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:15:20,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:20,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:20,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:20,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:20,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:20,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:20,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309065d1932d160344cb7833b2c833d245dd0_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031319818/Put/seqid=0 2023-09-06T20:15:20,474 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309065d1932d160344cb7833b2c833d245dd0_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309065d1932d160344cb7833b2c833d245dd0_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:20,475 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/697d816152824257a5feffdcfea2ee13, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:20,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/697d816152824257a5feffdcfea2ee13 is 175, key is test_row_0/A:col10/1694031319818/Put/seqid=0 2023-09-06T20:15:20,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031380543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:20,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031380648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:20,724 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/b38e705aca114e54bf989e9f3a3b4832 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b38e705aca114e54bf989e9f3a3b4832 2023-09-06T20:15:20,729 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/C of 3bb591e2a9ad6f7c9630808320336906 into b38e705aca114e54bf989e9f3a3b4832(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:20,729 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:20,729 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/C, priority=13, startTime=1694031320267; duration=0sec 2023-09-06T20:15:20,729 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:20,729 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:C 2023-09-06T20:15:20,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031380854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:20,882 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/697d816152824257a5feffdcfea2ee13 2023-09-06T20:15:20,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/aa3a65a30dda407aaec254cae390d7c6 is 50, key is test_row_0/B:col10/1694031319818/Put/seqid=0 2023-09-06T20:15:20,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/aa3a65a30dda407aaec254cae390d7c6 2023-09-06T20:15:20,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/b8e2556a47bd4172b2b7585b004cf435 is 50, key is test_row_0/C:col10/1694031319818/Put/seqid=0 2023-09-06T20:15:20,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/b8e2556a47bd4172b2b7585b004cf435 2023-09-06T20:15:20,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/697d816152824257a5feffdcfea2ee13 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/697d816152824257a5feffdcfea2ee13 2023-09-06T20:15:20,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/697d816152824257a5feffdcfea2ee13, entries=150, sequenceid=253, filesize=30.4 K 2023-09-06T20:15:20,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/aa3a65a30dda407aaec254cae390d7c6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/aa3a65a30dda407aaec254cae390d7c6 2023-09-06T20:15:20,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/aa3a65a30dda407aaec254cae390d7c6, entries=150, sequenceid=253, filesize=11.8 K 2023-09-06T20:15:20,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/b8e2556a47bd4172b2b7585b004cf435 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b8e2556a47bd4172b2b7585b004cf435 2023-09-06T20:15:20,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b8e2556a47bd4172b2b7585b004cf435, entries=150, sequenceid=253, filesize=11.8 K 2023-09-06T20:15:20,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3bb591e2a9ad6f7c9630808320336906 in 490ms, sequenceid=253, compaction requested=false 2023-09-06T20:15:20,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-06T20:15:21,042 INFO [Thread-2073 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 117 completed 2023-09-06T20:15:21,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2023-09-06T20:15:21,046 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:21,046 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-06T20:15:21,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-06T20:15:21,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:21,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-06T20:15:21,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:21,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:21,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:21,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:21,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:21,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:21,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309065871ad70a7174316b3d67c89282ca145_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031320528/Put/seqid=0 2023-09-06T20:15:21,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2023-09-06T20:15:21,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:21,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:21,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:21,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(53): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031381200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:21,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:21,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031381304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:21,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-06T20:15:21,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2023-09-06T20:15:21,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:21,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:21,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:21,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(53): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2023-09-06T20:15:21,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:21,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:21,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:21,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(53): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031381506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:21,584 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309065871ad70a7174316b3d67c89282ca145_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309065871ad70a7174316b3d67c89282ca145_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:21,585 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/960341be83dc44229a668e6e149ef8f9, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:21,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/960341be83dc44229a668e6e149ef8f9 is 175, key is test_row_0/A:col10/1694031320528/Put/seqid=0 2023-09-06T20:15:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-06T20:15:21,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2023-09-06T20:15:21,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:21,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:21,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:21,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(53): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2023-09-06T20:15:21,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:21,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:21,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:21,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(53): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031381812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:21,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2023-09-06T20:15:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:21,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(53): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:21,990 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/960341be83dc44229a668e6e149ef8f9 2023-09-06T20:15:21,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/8c59aca39120486c85dd8b61a5c4ec92 is 50, key is test_row_0/B:col10/1694031320528/Put/seqid=0 2023-09-06T20:15:22,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2023-09-06T20:15:22,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:22,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:22,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:22,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(53): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:22,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-06T20:15:22,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2023-09-06T20:15:22,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:22,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:22,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:22,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(53): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:22,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:22,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031382318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:22,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/8c59aca39120486c85dd8b61a5c4ec92 2023-09-06T20:15:22,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/b5bc5155545a43cfb173b33b03b3d88b is 50, key is test_row_0/C:col10/1694031320528/Put/seqid=0 2023-09-06T20:15:22,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2023-09-06T20:15:22,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:22,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:22,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:22,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(53): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:22,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:22,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/b5bc5155545a43cfb173b33b03b3d88b 2023-09-06T20:15:22,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/960341be83dc44229a668e6e149ef8f9 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/960341be83dc44229a668e6e149ef8f9 2023-09-06T20:15:22,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/960341be83dc44229a668e6e149ef8f9, entries=200, sequenceid=279, filesize=39.0 K 2023-09-06T20:15:22,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/8c59aca39120486c85dd8b61a5c4ec92 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/8c59aca39120486c85dd8b61a5c4ec92 2023-09-06T20:15:22,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/8c59aca39120486c85dd8b61a5c4ec92, entries=150, sequenceid=279, filesize=12.0 K 2023-09-06T20:15:22,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/b5bc5155545a43cfb173b33b03b3d88b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b5bc5155545a43cfb173b33b03b3d88b 2023-09-06T20:15:22,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b5bc5155545a43cfb173b33b03b3d88b, entries=150, sequenceid=279, filesize=12.0 K 2023-09-06T20:15:22,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3bb591e2a9ad6f7c9630808320336906 in 1293ms, sequenceid=279, compaction requested=true 2023-09-06T20:15:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:22,458 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:22,458 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:22,460 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37098 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:22,460 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102654 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:22,460 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/B is initiating minor compaction (all files) 2023-09-06T20:15:22,460 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/A is initiating minor compaction (all files) 2023-09-06T20:15:22,460 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/B in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:22,460 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/A in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:22,460 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/5623da743d5b417781fa64d34b31e113, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/697d816152824257a5feffdcfea2ee13, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/960341be83dc44229a668e6e149ef8f9] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=100.2 K 2023-09-06T20:15:22,460 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/cf82e62b11a0457693f1ac56cbefe42a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/aa3a65a30dda407aaec254cae390d7c6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/8c59aca39120486c85dd8b61a5c4ec92] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=36.2 K 2023-09-06T20:15:22,460 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:22,461 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/5623da743d5b417781fa64d34b31e113, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/697d816152824257a5feffdcfea2ee13, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/960341be83dc44229a668e6e149ef8f9] 2023-09-06T20:15:22,462 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting cf82e62b11a0457693f1ac56cbefe42a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1694031319161 2023-09-06T20:15:22,462 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5623da743d5b417781fa64d34b31e113, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1694031319161 2023-09-06T20:15:22,463 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting aa3a65a30dda407aaec254cae390d7c6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1694031319800 2023-09-06T20:15:22,463 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 8c59aca39120486c85dd8b61a5c4ec92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1694031320528 2023-09-06T20:15:22,463 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 697d816152824257a5feffdcfea2ee13, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1694031319800 2023-09-06T20:15:22,464 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 960341be83dc44229a668e6e149ef8f9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1694031320528 2023-09-06T20:15:22,473 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#B#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:22,475 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/cfdd61a3371d47e299e257bc264cd742 is 50, key is test_row_0/B:col10/1694031320528/Put/seqid=0 2023-09-06T20:15:22,482 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:22,507 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230906a5a6ac07cd5347109eb1007fd2180efb_3bb591e2a9ad6f7c9630808320336906 store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:22,508 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230906a5a6ac07cd5347109eb1007fd2180efb_3bb591e2a9ad6f7c9630808320336906, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:22,508 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906a5a6ac07cd5347109eb1007fd2180efb_3bb591e2a9ad6f7c9630808320336906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:22,537 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/cfdd61a3371d47e299e257bc264cd742 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/cfdd61a3371d47e299e257bc264cd742 2023-09-06T20:15:22,543 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/B of 3bb591e2a9ad6f7c9630808320336906 into cfdd61a3371d47e299e257bc264cd742(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:22,543 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:22,543 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/B, priority=13, startTime=1694031322458; duration=0sec 2023-09-06T20:15:22,543 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:22,543 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:B 2023-09-06T20:15:22,543 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:22,546 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37098 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:22,546 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/C is initiating minor compaction (all files) 2023-09-06T20:15:22,546 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/C in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:22,546 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b38e705aca114e54bf989e9f3a3b4832, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b8e2556a47bd4172b2b7585b004cf435, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b5bc5155545a43cfb173b33b03b3d88b] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=36.2 K 2023-09-06T20:15:22,547 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b38e705aca114e54bf989e9f3a3b4832, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1694031319161 2023-09-06T20:15:22,548 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b8e2556a47bd4172b2b7585b004cf435, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1694031319800 2023-09-06T20:15:22,548 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b5bc5155545a43cfb173b33b03b3d88b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1694031320528 2023-09-06T20:15:22,555 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#C#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:22,556 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/1960564528a5498fb71a8be63d7d53b1 is 50, key is test_row_0/C:col10/1694031320528/Put/seqid=0 2023-09-06T20:15:22,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2023-09-06T20:15:22,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:22,582 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:15:22,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:22,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:22,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:22,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:22,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:22,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:22,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090613e19a7d37c14faebc19c36199d1b221_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031321199/Put/seqid=0 2023-09-06T20:15:22,931 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#A#compaction#395 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:22,932 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/0d11ff34bd564494acfef6ad3b3656b5 is 175, key is test_row_0/A:col10/1694031320528/Put/seqid=0 2023-09-06T20:15:22,968 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/1960564528a5498fb71a8be63d7d53b1 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/1960564528a5498fb71a8be63d7d53b1 2023-09-06T20:15:22,976 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/C of 3bb591e2a9ad6f7c9630808320336906 into 1960564528a5498fb71a8be63d7d53b1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:22,976 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:22,976 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/C, priority=13, startTime=1694031322458; duration=0sec 2023-09-06T20:15:22,976 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:22,976 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:C 2023-09-06T20:15:23,020 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090613e19a7d37c14faebc19c36199d1b221_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090613e19a7d37c14faebc19c36199d1b221_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:23,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/242268d5d3294843bd7e23f8f5037298, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:23,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/242268d5d3294843bd7e23f8f5037298 is 175, key is test_row_0/A:col10/1694031321199/Put/seqid=0 2023-09-06T20:15:23,034 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/242268d5d3294843bd7e23f8f5037298 2023-09-06T20:15:23,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/65f297d8a87641d5935e29e7f62de56c is 50, key is test_row_0/B:col10/1694031321199/Put/seqid=0 2023-09-06T20:15:23,049 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/65f297d8a87641d5935e29e7f62de56c 2023-09-06T20:15:23,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/7e81d090cb3b4482abec74e13449669a is 50, key is test_row_0/C:col10/1694031321199/Put/seqid=0 2023-09-06T20:15:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-06T20:15:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:23,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:23,345 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/0d11ff34bd564494acfef6ad3b3656b5 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/0d11ff34bd564494acfef6ad3b3656b5 2023-09-06T20:15:23,350 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/A of 3bb591e2a9ad6f7c9630808320336906 into 0d11ff34bd564494acfef6ad3b3656b5(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:23,350 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:23,350 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/A, priority=13, startTime=1694031322458; duration=0sec 2023-09-06T20:15:23,350 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:23,350 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:A 2023-09-06T20:15:23,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031383348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031383349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031383350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031383350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031383355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031383456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031383456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031383456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031383456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031383459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,463 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/7e81d090cb3b4482abec74e13449669a 2023-09-06T20:15:23,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/242268d5d3294843bd7e23f8f5037298 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/242268d5d3294843bd7e23f8f5037298 2023-09-06T20:15:23,472 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/242268d5d3294843bd7e23f8f5037298, entries=150, sequenceid=291, filesize=30.5 K 2023-09-06T20:15:23,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/65f297d8a87641d5935e29e7f62de56c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/65f297d8a87641d5935e29e7f62de56c 2023-09-06T20:15:23,477 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/65f297d8a87641d5935e29e7f62de56c, entries=150, sequenceid=291, filesize=12.0 K 2023-09-06T20:15:23,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/7e81d090cb3b4482abec74e13449669a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/7e81d090cb3b4482abec74e13449669a 2023-09-06T20:15:23,486 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/7e81d090cb3b4482abec74e13449669a, entries=150, sequenceid=291, filesize=12.0 K 2023-09-06T20:15:23,487 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3bb591e2a9ad6f7c9630808320336906 in 905ms, sequenceid=291, compaction requested=false 2023-09-06T20:15:23,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:23,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:23,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=120 2023-09-06T20:15:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=120 2023-09-06T20:15:23,489 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=120, resume processing ppid=119 2023-09-06T20:15:23,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=120, ppid=119, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4420 sec 2023-09-06T20:15:23,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.4470 sec 2023-09-06T20:15:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:23,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-06T20:15:23,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:23,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:23,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:23,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:23,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:23,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:23,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906f65580c7dd94478eb4b14dd60b317ac6_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031323345/Put/seqid=0 2023-09-06T20:15:23,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031383667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031383668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031383670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031383672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031383674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031383775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031383776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031383777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031383779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031383781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031383978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031383978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031383980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031383983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:23,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031383986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:24,108 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906f65580c7dd94478eb4b14dd60b317ac6_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f65580c7dd94478eb4b14dd60b317ac6_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:24,109 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/83584b3d3f61447fa79ad1f0b718c8b1, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:24,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/83584b3d3f61447fa79ad1f0b718c8b1 is 175, key is test_row_0/A:col10/1694031323345/Put/seqid=0 2023-09-06T20:15:24,124 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/83584b3d3f61447fa79ad1f0b718c8b1 2023-09-06T20:15:24,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/e440f9a6d9134e94bc9ecdac2c1fd894 is 50, key is test_row_0/B:col10/1694031323345/Put/seqid=0 2023-09-06T20:15:24,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031384282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:24,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031384284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:24,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031384289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:24,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031384289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:24,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031384291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:24,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/e440f9a6d9134e94bc9ecdac2c1fd894 2023-09-06T20:15:24,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/f92862ae76f5498fa0903715afbbb40d is 50, key is test_row_0/C:col10/1694031323345/Put/seqid=0 2023-09-06T20:15:24,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031384790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:24,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031384792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:24,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031384796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:24,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031384797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:24,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031384801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:24,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/f92862ae76f5498fa0903715afbbb40d 2023-09-06T20:15:24,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/83584b3d3f61447fa79ad1f0b718c8b1 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/83584b3d3f61447fa79ad1f0b718c8b1 2023-09-06T20:15:24,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/83584b3d3f61447fa79ad1f0b718c8b1, entries=200, sequenceid=319, filesize=39.0 K 2023-09-06T20:15:24,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/e440f9a6d9134e94bc9ecdac2c1fd894 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/e440f9a6d9134e94bc9ecdac2c1fd894 2023-09-06T20:15:24,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/e440f9a6d9134e94bc9ecdac2c1fd894, entries=150, sequenceid=319, filesize=12.0 K 2023-09-06T20:15:24,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/f92862ae76f5498fa0903715afbbb40d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f92862ae76f5498fa0903715afbbb40d 2023-09-06T20:15:24,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f92862ae76f5498fa0903715afbbb40d, entries=150, sequenceid=319, filesize=12.0 K 2023-09-06T20:15:24,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 3bb591e2a9ad6f7c9630808320336906 in 1314ms, sequenceid=319, compaction requested=true 2023-09-06T20:15:24,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:24,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:24,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:24,978 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:24,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:24,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:24,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:24,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-06T20:15:24,978 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:24,980 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37500 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:24,980 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/B is initiating minor compaction (all files) 2023-09-06T20:15:24,980 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/B in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:24,980 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/cfdd61a3371d47e299e257bc264cd742, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/65f297d8a87641d5935e29e7f62de56c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/e440f9a6d9134e94bc9ecdac2c1fd894] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=36.6 K 2023-09-06T20:15:24,981 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103056 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:24,981 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/A is initiating minor compaction (all files) 2023-09-06T20:15:24,981 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/A in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:24,981 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting cfdd61a3371d47e299e257bc264cd742, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1694031320528 2023-09-06T20:15:24,981 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/0d11ff34bd564494acfef6ad3b3656b5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/242268d5d3294843bd7e23f8f5037298, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/83584b3d3f61447fa79ad1f0b718c8b1] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=100.6 K 2023-09-06T20:15:24,981 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:24,981 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/0d11ff34bd564494acfef6ad3b3656b5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/242268d5d3294843bd7e23f8f5037298, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/83584b3d3f61447fa79ad1f0b718c8b1] 2023-09-06T20:15:24,982 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 0d11ff34bd564494acfef6ad3b3656b5, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1694031320528 2023-09-06T20:15:24,982 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 65f297d8a87641d5935e29e7f62de56c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1694031321174 2023-09-06T20:15:24,982 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 242268d5d3294843bd7e23f8f5037298, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1694031321174 2023-09-06T20:15:24,982 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting e440f9a6d9134e94bc9ecdac2c1fd894, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1694031323345 2023-09-06T20:15:24,982 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 83584b3d3f61447fa79ad1f0b718c8b1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1694031323345 2023-09-06T20:15:25,007 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:25,011 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#B#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:25,012 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/25bb147939b741c0b7edce3ba6a35b97 is 50, key is test_row_0/B:col10/1694031323345/Put/seqid=0 2023-09-06T20:15:25,012 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230906764987c86955454b88655eae881832cf_3bb591e2a9ad6f7c9630808320336906 store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:25,013 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230906764987c86955454b88655eae881832cf_3bb591e2a9ad6f7c9630808320336906, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:25,014 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906764987c86955454b88655eae881832cf_3bb591e2a9ad6f7c9630808320336906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:25,044 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#A#compaction#403 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:25,045 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/833b5010886847e3b0a822f22ec1e8c8 is 175, key is test_row_0/A:col10/1694031323345/Put/seqid=0 2023-09-06T20:15:25,046 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/25bb147939b741c0b7edce3ba6a35b97 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/25bb147939b741c0b7edce3ba6a35b97 2023-09-06T20:15:25,050 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/B of 3bb591e2a9ad6f7c9630808320336906 into 25bb147939b741c0b7edce3ba6a35b97(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:25,050 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:25,050 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/B, priority=13, startTime=1694031324978; duration=0sec 2023-09-06T20:15:25,050 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:25,050 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:B 2023-09-06T20:15:25,050 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:25,051 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37500 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:25,051 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/C is initiating minor compaction (all files) 2023-09-06T20:15:25,051 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/C in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:25,051 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/1960564528a5498fb71a8be63d7d53b1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/7e81d090cb3b4482abec74e13449669a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f92862ae76f5498fa0903715afbbb40d] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=36.6 K 2023-09-06T20:15:25,052 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1960564528a5498fb71a8be63d7d53b1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1694031320528 2023-09-06T20:15:25,053 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7e81d090cb3b4482abec74e13449669a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1694031321174 2023-09-06T20:15:25,053 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting f92862ae76f5498fa0903715afbbb40d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1694031323345 2023-09-06T20:15:25,060 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/833b5010886847e3b0a822f22ec1e8c8 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/833b5010886847e3b0a822f22ec1e8c8 2023-09-06T20:15:25,060 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#C#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:25,060 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/f1bc94c5f1ee41279b9797d03903e751 is 50, key is test_row_0/C:col10/1694031323345/Put/seqid=0 2023-09-06T20:15:25,064 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/A of 3bb591e2a9ad6f7c9630808320336906 into 833b5010886847e3b0a822f22ec1e8c8(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:25,065 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:25,065 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/A, priority=13, startTime=1694031324978; duration=0sec 2023-09-06T20:15:25,065 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:25,065 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:A 2023-09-06T20:15:25,078 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/f1bc94c5f1ee41279b9797d03903e751 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f1bc94c5f1ee41279b9797d03903e751 2023-09-06T20:15:25,082 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/C of 3bb591e2a9ad6f7c9630808320336906 into f1bc94c5f1ee41279b9797d03903e751(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:25,082 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:25,082 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/C, priority=13, startTime=1694031324978; duration=0sec 2023-09-06T20:15:25,082 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:25,082 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:C 2023-09-06T20:15:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-06T20:15:25,151 INFO [Thread-2073 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 119 completed 2023-09-06T20:15:25,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2023-09-06T20:15:25,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-09-06T20:15:25,155 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:25,155 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:25,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-09-06T20:15:25,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2023-09-06T20:15:25,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:25,308 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2023-09-06T20:15:25,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:25,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:25,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:25,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:25,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:25,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:25,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906f3ca23f3e72c49e189184bae6170b80c_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031323666/Put/seqid=0 2023-09-06T20:15:25,326 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906f3ca23f3e72c49e189184bae6170b80c_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f3ca23f3e72c49e189184bae6170b80c_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:25,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/f39f370e9ad4481cac6525108ba489c2, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:25,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/f39f370e9ad4481cac6525108ba489c2 is 175, key is test_row_0/A:col10/1694031323666/Put/seqid=0 2023-09-06T20:15:25,332 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/f39f370e9ad4481cac6525108ba489c2 2023-09-06T20:15:25,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/349973206d1f42aab1a3d38fe8460305 is 50, key is test_row_0/B:col10/1694031323666/Put/seqid=0 2023-09-06T20:15:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-09-06T20:15:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-09-06T20:15:25,776 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/349973206d1f42aab1a3d38fe8460305 2023-09-06T20:15:25,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/95f0db1297c14956a5943a07e84a609d is 50, key is test_row_0/C:col10/1694031323666/Put/seqid=0 2023-09-06T20:15:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:25,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:25,803 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/95f0db1297c14956a5943a07e84a609d 2023-09-06T20:15:25,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/f39f370e9ad4481cac6525108ba489c2 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/f39f370e9ad4481cac6525108ba489c2 2023-09-06T20:15:25,812 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/f39f370e9ad4481cac6525108ba489c2, entries=150, sequenceid=332, filesize=30.5 K 2023-09-06T20:15:25,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/349973206d1f42aab1a3d38fe8460305 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/349973206d1f42aab1a3d38fe8460305 2023-09-06T20:15:25,816 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/349973206d1f42aab1a3d38fe8460305, entries=150, sequenceid=332, filesize=12.0 K 2023-09-06T20:15:25,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/95f0db1297c14956a5943a07e84a609d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/95f0db1297c14956a5943a07e84a609d 2023-09-06T20:15:25,821 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/95f0db1297c14956a5943a07e84a609d, entries=150, sequenceid=332, filesize=12.0 K 2023-09-06T20:15:25,822 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3022): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=73.80 KB/75570 for 3bb591e2a9ad6f7c9630808320336906 in 515ms, sequenceid=332, compaction requested=false 2023-09-06T20:15:25,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:25,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:25,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=122 2023-09-06T20:15:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=122 2023-09-06T20:15:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:25,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-06T20:15:25,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=122, resume processing ppid=121 2023-09-06T20:15:25,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=122, ppid=121, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 668 msec 2023-09-06T20:15:25,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:25,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:25,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:25,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:25,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:25,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:25,827 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=121, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 672 msec 2023-09-06T20:15:25,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906a200042fe8c145b883602377ae483f1d_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031325824/Put/seqid=0 2023-09-06T20:15:25,849 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906a200042fe8c145b883602377ae483f1d_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906a200042fe8c145b883602377ae483f1d_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:25,849 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/d28ad01e1090432c9b10cac08c782b64, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:25,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/d28ad01e1090432c9b10cac08c782b64 is 175, key is test_row_0/A:col10/1694031325824/Put/seqid=0 2023-09-06T20:15:25,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031385850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:25,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031385850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:25,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:25,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031385851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031385851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:25,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031385851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:25,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031385959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:25,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031385959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:25,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031385959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:25,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031385959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:25,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031385959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:26,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031386163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:26,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031386164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:26,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031386164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:26,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031386165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:26,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031386165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:26,255 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/d28ad01e1090432c9b10cac08c782b64 2023-09-06T20:15:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-09-06T20:15:26,258 INFO [Thread-2073 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 121 completed 2023-09-06T20:15:26,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/b8fdd80ee14f4d9bb8a8f330ec822527 is 50, key is test_row_0/B:col10/1694031325824/Put/seqid=0 2023-09-06T20:15:26,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2023-09-06T20:15:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=123 2023-09-06T20:15:26,268 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:26,269 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:26,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=123 2023-09-06T20:15:26,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2023-09-06T20:15:26,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:26,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:26,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:26,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(53): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:26,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:26,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46374 deadline: 1694031386468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:26,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46320 deadline: 1694031386469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:26,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46334 deadline: 1694031386469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:26,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46322 deadline: 1694031386470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:26,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:46362 deadline: 1694031386471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=123 2023-09-06T20:15:26,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2023-09-06T20:15:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:26,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(53): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:26,586 DEBUG [Thread-2082 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3e197663 to 127.0.0.1:58481 2023-09-06T20:15:26,587 DEBUG [Thread-2082 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:26,588 DEBUG [Thread-2078 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x08e0c80d to 127.0.0.1:58481 2023-09-06T20:15:26,588 DEBUG [Thread-2076 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x62a48015 to 127.0.0.1:58481 2023-09-06T20:15:26,588 DEBUG [Thread-2078 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:26,588 DEBUG [Thread-2076 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:26,592 DEBUG [Thread-2074 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5a3691cd to 127.0.0.1:58481 2023-09-06T20:15:26,592 DEBUG [Thread-2074 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:26,594 DEBUG [Thread-2080 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x359fa346 to 127.0.0.1:58481 2023-09-06T20:15:26,594 DEBUG [Thread-2080 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:26,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/b8fdd80ee14f4d9bb8a8f330ec822527 2023-09-06T20:15:26,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/409f0593853049b6866fa86af8fb86c0 is 50, key is test_row_0/C:col10/1694031325824/Put/seqid=0 2023-09-06T20:15:26,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/409f0593853049b6866fa86af8fb86c0 2023-09-06T20:15:26,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/d28ad01e1090432c9b10cac08c782b64 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/d28ad01e1090432c9b10cac08c782b64 2023-09-06T20:15:26,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/d28ad01e1090432c9b10cac08c782b64, entries=250, sequenceid=348, filesize=47.5 K 2023-09-06T20:15:26,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/b8fdd80ee14f4d9bb8a8f330ec822527 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b8fdd80ee14f4d9bb8a8f330ec822527 2023-09-06T20:15:26,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b8fdd80ee14f4d9bb8a8f330ec822527, entries=150, sequenceid=348, filesize=12.0 K 2023-09-06T20:15:26,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/409f0593853049b6866fa86af8fb86c0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/409f0593853049b6866fa86af8fb86c0 2023-09-06T20:15:26,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/409f0593853049b6866fa86af8fb86c0, entries=150, sequenceid=348, filesize=12.0 K 2023-09-06T20:15:26,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 3bb591e2a9ad6f7c9630808320336906 in 872ms, sequenceid=348, compaction requested=true 2023-09-06T20:15:26,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:26,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:26,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:26,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:26,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:26,696 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:26,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bb591e2a9ad6f7c9630808320336906:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:26,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:26,696 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:26,697 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111848 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:26,697 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37602 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:26,697 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/A is initiating minor compaction (all files) 2023-09-06T20:15:26,697 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/B is initiating minor compaction (all files) 2023-09-06T20:15:26,697 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/B in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:26,697 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/A in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:26,697 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/25bb147939b741c0b7edce3ba6a35b97, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/349973206d1f42aab1a3d38fe8460305, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b8fdd80ee14f4d9bb8a8f330ec822527] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=36.7 K 2023-09-06T20:15:26,697 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/833b5010886847e3b0a822f22ec1e8c8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/f39f370e9ad4481cac6525108ba489c2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/d28ad01e1090432c9b10cac08c782b64] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=109.2 K 2023-09-06T20:15:26,697 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:26,697 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/833b5010886847e3b0a822f22ec1e8c8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/f39f370e9ad4481cac6525108ba489c2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/d28ad01e1090432c9b10cac08c782b64] 2023-09-06T20:15:26,698 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 25bb147939b741c0b7edce3ba6a35b97, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1694031323345 2023-09-06T20:15:26,698 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 833b5010886847e3b0a822f22ec1e8c8, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1694031323345 2023-09-06T20:15:26,698 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 349973206d1f42aab1a3d38fe8460305, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1694031323666 2023-09-06T20:15:26,698 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting f39f370e9ad4481cac6525108ba489c2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1694031323666 2023-09-06T20:15:26,698 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b8fdd80ee14f4d9bb8a8f330ec822527, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1694031325816 2023-09-06T20:15:26,698 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting d28ad01e1090432c9b10cac08c782b64, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1694031325801 2023-09-06T20:15:26,708 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#B#compaction#412 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:26,708 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/20ca81cd375a44e2bc6c32a156d8f6e3 is 50, key is test_row_0/B:col10/1694031325824/Put/seqid=0 2023-09-06T20:15:26,709 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:26,712 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2023090682477574a8fd43c48de28bbdafb48f36_3bb591e2a9ad6f7c9630808320336906 store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:26,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2023-09-06T20:15:26,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:26,727 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2023-09-06T20:15:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:26,734 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2023090682477574a8fd43c48de28bbdafb48f36_3bb591e2a9ad6f7c9630808320336906, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:26,734 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090682477574a8fd43c48de28bbdafb48f36_3bb591e2a9ad6f7c9630808320336906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:26,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c7694ad2f72a43c2a0f0b095d007e3e1_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031325850/Put/seqid=0 2023-09-06T20:15:26,752 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c7694ad2f72a43c2a0f0b095d007e3e1_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c7694ad2f72a43c2a0f0b095d007e3e1_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:26,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/e188de3d4684491190c4d04286f7db0d, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:26,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/e188de3d4684491190c4d04286f7db0d is 175, key is test_row_0/A:col10/1694031325850/Put/seqid=0 2023-09-06T20:15:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=123 2023-09-06T20:15:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:26,977 DEBUG [Thread-2067 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x48325705 to 127.0.0.1:58481 2023-09-06T20:15:26,977 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:26,977 DEBUG [Thread-2071 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4fe89067 to 127.0.0.1:58481 2023-09-06T20:15:26,977 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:26,979 DEBUG [Thread-2065 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2e6d9e88 to 127.0.0.1:58481 2023-09-06T20:15:26,979 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. as already flushing 2023-09-06T20:15:26,980 DEBUG [Thread-2063 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x278f4124 to 127.0.0.1:58481 2023-09-06T20:15:26,980 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:26,982 DEBUG [Thread-2069 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x64069dd3 to 127.0.0.1:58481 2023-09-06T20:15:26,982 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:27,116 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/20ca81cd375a44e2bc6c32a156d8f6e3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/20ca81cd375a44e2bc6c32a156d8f6e3 2023-09-06T20:15:27,119 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/B of 3bb591e2a9ad6f7c9630808320336906 into 20ca81cd375a44e2bc6c32a156d8f6e3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:27,119 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:27,119 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/B, priority=13, startTime=1694031326696; duration=0sec 2023-09-06T20:15:27,119 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:27,119 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:B 2023-09-06T20:15:27,119 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:27,120 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37602 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:27,120 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 3bb591e2a9ad6f7c9630808320336906/C is initiating minor compaction (all files) 2023-09-06T20:15:27,120 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 3bb591e2a9ad6f7c9630808320336906/C in TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:27,120 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f1bc94c5f1ee41279b9797d03903e751, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/95f0db1297c14956a5943a07e84a609d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/409f0593853049b6866fa86af8fb86c0] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp, totalSize=36.7 K 2023-09-06T20:15:27,121 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting f1bc94c5f1ee41279b9797d03903e751, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1694031323345 2023-09-06T20:15:27,121 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 95f0db1297c14956a5943a07e84a609d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1694031323666 2023-09-06T20:15:27,122 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 409f0593853049b6866fa86af8fb86c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1694031325816 2023-09-06T20:15:27,127 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#C#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:27,128 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/d30b0a56ebdd464cae488dd077806650 is 50, key is test_row_0/C:col10/1694031325824/Put/seqid=0 2023-09-06T20:15:27,140 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bb591e2a9ad6f7c9630808320336906#A#compaction#413 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:27,141 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/c97d8a74c9a64d2caf1703b9d20d3fc4 is 175, key is test_row_0/A:col10/1694031325824/Put/seqid=0 2023-09-06T20:15:27,160 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/e188de3d4684491190c4d04286f7db0d 2023-09-06T20:15:27,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/d8a69f90faa64a6697ac657e0079be5b is 50, key is test_row_0/B:col10/1694031325850/Put/seqid=0 2023-09-06T20:15:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=123 2023-09-06T20:15:27,537 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/d30b0a56ebdd464cae488dd077806650 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/d30b0a56ebdd464cae488dd077806650 2023-09-06T20:15:27,541 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/C of 3bb591e2a9ad6f7c9630808320336906 into d30b0a56ebdd464cae488dd077806650(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:27,541 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:27,541 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/C, priority=13, startTime=1694031326696; duration=0sec 2023-09-06T20:15:27,541 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:27,541 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:C 2023-09-06T20:15:27,547 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/c97d8a74c9a64d2caf1703b9d20d3fc4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/c97d8a74c9a64d2caf1703b9d20d3fc4 2023-09-06T20:15:27,551 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 3bb591e2a9ad6f7c9630808320336906/A of 3bb591e2a9ad6f7c9630808320336906 into c97d8a74c9a64d2caf1703b9d20d3fc4(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:27,551 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:27,551 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906., storeName=3bb591e2a9ad6f7c9630808320336906/A, priority=13, startTime=1694031326696; duration=0sec 2023-09-06T20:15:27,551 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:27,551 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bb591e2a9ad6f7c9630808320336906:A 2023-09-06T20:15:27,569 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/d8a69f90faa64a6697ac657e0079be5b 2023-09-06T20:15:27,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/eb4ebdf7f4424e1bb6faaad7ec316584 is 50, key is test_row_0/C:col10/1694031325850/Put/seqid=0 2023-09-06T20:15:27,978 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/eb4ebdf7f4424e1bb6faaad7ec316584 2023-09-06T20:15:27,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/e188de3d4684491190c4d04286f7db0d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/e188de3d4684491190c4d04286f7db0d 2023-09-06T20:15:27,984 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/e188de3d4684491190c4d04286f7db0d, entries=150, sequenceid=369, filesize=30.5 K 2023-09-06T20:15:27,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/d8a69f90faa64a6697ac657e0079be5b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/d8a69f90faa64a6697ac657e0079be5b 2023-09-06T20:15:27,988 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/d8a69f90faa64a6697ac657e0079be5b, entries=150, sequenceid=369, filesize=12.0 K 2023-09-06T20:15:27,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/eb4ebdf7f4424e1bb6faaad7ec316584 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/eb4ebdf7f4424e1bb6faaad7ec316584 2023-09-06T20:15:27,992 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/eb4ebdf7f4424e1bb6faaad7ec316584, entries=150, sequenceid=369, filesize=12.0 K 2023-09-06T20:15:27,993 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3022): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=33.54 KB/34350 for 3bb591e2a9ad6f7c9630808320336906 in 1266ms, sequenceid=369, compaction requested=false 2023-09-06T20:15:27,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2520): Flush status journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:27,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:27,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=124 2023-09-06T20:15:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=124 2023-09-06T20:15:27,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=124, resume processing ppid=123 2023-09-06T20:15:27,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=124, ppid=123, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7250 sec 2023-09-06T20:15:27,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=123, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.7310 sec 2023-09-06T20:15:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=123 2023-09-06T20:15:28,369 INFO [Thread-2073 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 123 completed 2023-09-06T20:15:28,369 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2023-09-06T20:15:28,369 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2023-09-06T20:15:28,369 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2023-09-06T20:15:28,369 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(402): scanned 2601 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(403): verified 7802 rows 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(402): scanned 2615 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(403): verified 7845 rows 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(402): scanned 2625 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(403): verified 7875 rows 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(402): scanned 2629 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(403): verified 7886 rows 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(402): scanned 2614 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(403): verified 7842 rows 2023-09-06T20:15:28,370 INFO [Listener at localhost/35961 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:15:28,370 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x41c47eb6 to 127.0.0.1:58481 2023-09-06T20:15:28,370 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:28,372 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2023-09-06T20:15:28,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable TestAcidGuarantees 2023-09-06T20:15:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=125, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=125 2023-09-06T20:15:28,376 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031328376"}]},"ts":"1694031328376"} 2023-09-06T20:15:28,377 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2023-09-06T20:15:28,379 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2023-09-06T20:15:28,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bb591e2a9ad6f7c9630808320336906, UNASSIGN}] 2023-09-06T20:15:28,382 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=126, ppid=125, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bb591e2a9ad6f7c9630808320336906, UNASSIGN 2023-09-06T20:15:28,383 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=126 updating hbase:meta row=3bb591e2a9ad6f7c9630808320336906, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:28,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE; CloseRegionProcedure 3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:15:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=125 2023-09-06T20:15:28,535 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] handler.UnassignRegionHandler(119): Close 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:28,535 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(1668): Closing 3bb591e2a9ad6f7c9630808320336906, disabling compactions & flushes 2023-09-06T20:15:28,535 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:28,535 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:28,535 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. after waiting 0 ms 2023-09-06T20:15:28,535 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:28,535 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(2819): Flushing 3bb591e2a9ad6f7c9630808320336906 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2023-09-06T20:15:28,535 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=A 2023-09-06T20:15:28,536 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:28,536 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=B 2023-09-06T20:15:28,536 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:28,536 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 3bb591e2a9ad6f7c9630808320336906, store=C 2023-09-06T20:15:28,536 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:28,540 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309060b4d66dbb5a949fea444a1909837e9c2_3bb591e2a9ad6f7c9630808320336906 is 50, key is test_row_0/A:col10/1694031326977/Put/seqid=0 2023-09-06T20:15:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=125 2023-09-06T20:15:28,951 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309060b4d66dbb5a949fea444a1909837e9c2_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309060b4d66dbb5a949fea444a1909837e9c2_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:28,951 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/88536388725c4451b4697b8b16ecc335, store: [table=TestAcidGuarantees family=A region=3bb591e2a9ad6f7c9630808320336906] 2023-09-06T20:15:28,952 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/88536388725c4451b4697b8b16ecc335 is 175, key is test_row_0/A:col10/1694031326977/Put/seqid=0 2023-09-06T20:15:28,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=125 2023-09-06T20:15:29,356 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/88536388725c4451b4697b8b16ecc335 2023-09-06T20:15:29,361 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/707c130a988c43edae75c5b3dd3e5f05 is 50, key is test_row_0/B:col10/1694031326977/Put/seqid=0 2023-09-06T20:15:29,365 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/707c130a988c43edae75c5b3dd3e5f05 2023-09-06T20:15:29,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/dd9532809bfc459a94da1336419ca898 is 50, key is test_row_0/C:col10/1694031326977/Put/seqid=0 2023-09-06T20:15:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=125 2023-09-06T20:15:29,773 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/dd9532809bfc459a94da1336419ca898 2023-09-06T20:15:29,777 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/A/88536388725c4451b4697b8b16ecc335 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/88536388725c4451b4697b8b16ecc335 2023-09-06T20:15:29,780 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/88536388725c4451b4697b8b16ecc335, entries=150, sequenceid=380, filesize=30.5 K 2023-09-06T20:15:29,780 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/B/707c130a988c43edae75c5b3dd3e5f05 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/707c130a988c43edae75c5b3dd3e5f05 2023-09-06T20:15:29,783 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/707c130a988c43edae75c5b3dd3e5f05, entries=150, sequenceid=380, filesize=12.0 K 2023-09-06T20:15:29,784 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/.tmp/C/dd9532809bfc459a94da1336419ca898 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/dd9532809bfc459a94da1336419ca898 2023-09-06T20:15:29,788 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/dd9532809bfc459a94da1336419ca898, entries=150, sequenceid=380, filesize=12.0 K 2023-09-06T20:15:29,788 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(3022): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3bb591e2a9ad6f7c9630808320336906 in 1253ms, sequenceid=380, compaction requested=true 2023-09-06T20:15:29,810 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/be7ea0f3937e4b42816582fdea17ad25, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/07c4e6a7deda4758b7e1d7674c5100a4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/1e83aecc549e4998a74a26f8525a5301, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/9b2beb5435044c91b57b8ab617c5786b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/815671cb5bea4ada8419c23b86a74a36, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8b7daf50efcc45d0b7b9f92e4673cdf7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8bf0763d6aea4d338890e3854a248344, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/e89263ec8daf48cca8b4108390e59974, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8ff24b28822349a6b28d18327392b571, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/5a81c87b04304990ab20a1331c37b756, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/cc2bf3a7adfe4d76a0ac7106df8d095a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/eb670e40f0484a2b8d5c2d52b58b05bd, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/1c3856a484744133925817978581b964, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/3ffa6678e292471c906e8becef4e3de5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/697c9bdaa023447ebff62e3262d24e5d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/dc1ff4d87fd043a7af1cd2dc9629b17b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/5623da743d5b417781fa64d34b31e113, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/697d816152824257a5feffdcfea2ee13, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/960341be83dc44229a668e6e149ef8f9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/0d11ff34bd564494acfef6ad3b3656b5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/242268d5d3294843bd7e23f8f5037298, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/83584b3d3f61447fa79ad1f0b718c8b1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/833b5010886847e3b0a822f22ec1e8c8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/f39f370e9ad4481cac6525108ba489c2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/d28ad01e1090432c9b10cac08c782b64] to archive 2023-09-06T20:15:29,811 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:15:29,813 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/be7ea0f3937e4b42816582fdea17ad25 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/be7ea0f3937e4b42816582fdea17ad25 2023-09-06T20:15:29,814 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/07c4e6a7deda4758b7e1d7674c5100a4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/07c4e6a7deda4758b7e1d7674c5100a4 2023-09-06T20:15:29,815 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/1e83aecc549e4998a74a26f8525a5301 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/1e83aecc549e4998a74a26f8525a5301 2023-09-06T20:15:29,815 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/9b2beb5435044c91b57b8ab617c5786b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/9b2beb5435044c91b57b8ab617c5786b 2023-09-06T20:15:29,816 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/815671cb5bea4ada8419c23b86a74a36 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/815671cb5bea4ada8419c23b86a74a36 2023-09-06T20:15:29,817 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8b7daf50efcc45d0b7b9f92e4673cdf7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8b7daf50efcc45d0b7b9f92e4673cdf7 2023-09-06T20:15:29,818 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8bf0763d6aea4d338890e3854a248344 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8bf0763d6aea4d338890e3854a248344 2023-09-06T20:15:29,819 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/e89263ec8daf48cca8b4108390e59974 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/e89263ec8daf48cca8b4108390e59974 2023-09-06T20:15:29,820 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8ff24b28822349a6b28d18327392b571 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/8ff24b28822349a6b28d18327392b571 2023-09-06T20:15:29,821 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/5a81c87b04304990ab20a1331c37b756 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/5a81c87b04304990ab20a1331c37b756 2023-09-06T20:15:29,822 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/cc2bf3a7adfe4d76a0ac7106df8d095a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/cc2bf3a7adfe4d76a0ac7106df8d095a 2023-09-06T20:15:29,823 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/eb670e40f0484a2b8d5c2d52b58b05bd to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/eb670e40f0484a2b8d5c2d52b58b05bd 2023-09-06T20:15:29,824 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/1c3856a484744133925817978581b964 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/1c3856a484744133925817978581b964 2023-09-06T20:15:29,825 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/3ffa6678e292471c906e8becef4e3de5 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/3ffa6678e292471c906e8becef4e3de5 2023-09-06T20:15:29,826 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/697c9bdaa023447ebff62e3262d24e5d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/697c9bdaa023447ebff62e3262d24e5d 2023-09-06T20:15:29,827 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/dc1ff4d87fd043a7af1cd2dc9629b17b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/dc1ff4d87fd043a7af1cd2dc9629b17b 2023-09-06T20:15:29,828 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/5623da743d5b417781fa64d34b31e113 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/5623da743d5b417781fa64d34b31e113 2023-09-06T20:15:29,828 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/697d816152824257a5feffdcfea2ee13 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/697d816152824257a5feffdcfea2ee13 2023-09-06T20:15:29,829 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/960341be83dc44229a668e6e149ef8f9 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/960341be83dc44229a668e6e149ef8f9 2023-09-06T20:15:29,830 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/0d11ff34bd564494acfef6ad3b3656b5 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/0d11ff34bd564494acfef6ad3b3656b5 2023-09-06T20:15:29,831 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/242268d5d3294843bd7e23f8f5037298 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/242268d5d3294843bd7e23f8f5037298 2023-09-06T20:15:29,832 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/83584b3d3f61447fa79ad1f0b718c8b1 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/83584b3d3f61447fa79ad1f0b718c8b1 2023-09-06T20:15:29,834 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/833b5010886847e3b0a822f22ec1e8c8 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/833b5010886847e3b0a822f22ec1e8c8 2023-09-06T20:15:29,835 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/f39f370e9ad4481cac6525108ba489c2 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/f39f370e9ad4481cac6525108ba489c2 2023-09-06T20:15:29,836 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/d28ad01e1090432c9b10cac08c782b64 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/d28ad01e1090432c9b10cac08c782b64 2023-09-06T20:15:29,838 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/401c069744d7450b9b128129fe5eb280, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/01cb204792e74a14a5c28542092f3c51, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/e61736ad7a484bc4b3df6cd9fc88f0f4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/7f34175be50f4d2eb4f9086290eccf2c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/13baf40becae4b31a96025838f9672f4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/1bba22018fdb4541af6e448c030304c1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/32ed6358ebca4f5192b8f15e46e36888, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b78832007f944cf5808c5793ab3d3fb1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/be96018f92f64e449536e62f8394053e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/843253acd4594764b5f06a0b80de2fa7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/04b1aba045c44717834b219b9931b8b9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/0c89b0fd5ccf452f85eb38c4e521e5f9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/dbe0b448b98d4162a9e95e99cf0180f5, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b8e8a335deaf481fb2be058efab45de7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/1ee75fd7993343359846142f11abd2d2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/cf82e62b11a0457693f1ac56cbefe42a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/2c2ea285254241c3b51f0f6018313eb8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/aa3a65a30dda407aaec254cae390d7c6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/cfdd61a3371d47e299e257bc264cd742, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/8c59aca39120486c85dd8b61a5c4ec92, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/65f297d8a87641d5935e29e7f62de56c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/25bb147939b741c0b7edce3ba6a35b97, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/e440f9a6d9134e94bc9ecdac2c1fd894, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/349973206d1f42aab1a3d38fe8460305, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b8fdd80ee14f4d9bb8a8f330ec822527] to archive 2023-09-06T20:15:29,839 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:15:29,840 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/401c069744d7450b9b128129fe5eb280 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/401c069744d7450b9b128129fe5eb280 2023-09-06T20:15:29,841 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/01cb204792e74a14a5c28542092f3c51 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/01cb204792e74a14a5c28542092f3c51 2023-09-06T20:15:29,842 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/e61736ad7a484bc4b3df6cd9fc88f0f4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/e61736ad7a484bc4b3df6cd9fc88f0f4 2023-09-06T20:15:29,843 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/7f34175be50f4d2eb4f9086290eccf2c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/7f34175be50f4d2eb4f9086290eccf2c 2023-09-06T20:15:29,844 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/13baf40becae4b31a96025838f9672f4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/13baf40becae4b31a96025838f9672f4 2023-09-06T20:15:29,846 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/1bba22018fdb4541af6e448c030304c1 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/1bba22018fdb4541af6e448c030304c1 2023-09-06T20:15:29,847 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/32ed6358ebca4f5192b8f15e46e36888 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/32ed6358ebca4f5192b8f15e46e36888 2023-09-06T20:15:29,848 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b78832007f944cf5808c5793ab3d3fb1 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b78832007f944cf5808c5793ab3d3fb1 2023-09-06T20:15:29,848 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/be96018f92f64e449536e62f8394053e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/be96018f92f64e449536e62f8394053e 2023-09-06T20:15:29,849 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/843253acd4594764b5f06a0b80de2fa7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/843253acd4594764b5f06a0b80de2fa7 2023-09-06T20:15:29,850 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/04b1aba045c44717834b219b9931b8b9 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/04b1aba045c44717834b219b9931b8b9 2023-09-06T20:15:29,851 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/0c89b0fd5ccf452f85eb38c4e521e5f9 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/0c89b0fd5ccf452f85eb38c4e521e5f9 2023-09-06T20:15:29,852 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/dbe0b448b98d4162a9e95e99cf0180f5 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/dbe0b448b98d4162a9e95e99cf0180f5 2023-09-06T20:15:29,853 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b8e8a335deaf481fb2be058efab45de7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b8e8a335deaf481fb2be058efab45de7 2023-09-06T20:15:29,853 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/1ee75fd7993343359846142f11abd2d2 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/1ee75fd7993343359846142f11abd2d2 2023-09-06T20:15:29,854 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/cf82e62b11a0457693f1ac56cbefe42a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/cf82e62b11a0457693f1ac56cbefe42a 2023-09-06T20:15:29,855 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/2c2ea285254241c3b51f0f6018313eb8 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/2c2ea285254241c3b51f0f6018313eb8 2023-09-06T20:15:29,856 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/aa3a65a30dda407aaec254cae390d7c6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/aa3a65a30dda407aaec254cae390d7c6 2023-09-06T20:15:29,857 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/cfdd61a3371d47e299e257bc264cd742 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/cfdd61a3371d47e299e257bc264cd742 2023-09-06T20:15:29,858 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/8c59aca39120486c85dd8b61a5c4ec92 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/8c59aca39120486c85dd8b61a5c4ec92 2023-09-06T20:15:29,859 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/65f297d8a87641d5935e29e7f62de56c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/65f297d8a87641d5935e29e7f62de56c 2023-09-06T20:15:29,860 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/25bb147939b741c0b7edce3ba6a35b97 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/25bb147939b741c0b7edce3ba6a35b97 2023-09-06T20:15:29,861 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/e440f9a6d9134e94bc9ecdac2c1fd894 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/e440f9a6d9134e94bc9ecdac2c1fd894 2023-09-06T20:15:29,862 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/349973206d1f42aab1a3d38fe8460305 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/349973206d1f42aab1a3d38fe8460305 2023-09-06T20:15:29,863 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b8fdd80ee14f4d9bb8a8f330ec822527 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/b8fdd80ee14f4d9bb8a8f330ec822527 2023-09-06T20:15:29,865 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/954b3a9bb66d4930ac5f3583ac717b3a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/bdbe6debf3504c34b1076b790d961106, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/96e221ae23e5401a8e159ec883dcf452, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/0661da3230f34d3faf6047111114679a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/5d889ee1955d4b8fa079239a928e6cb7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f76895f6b7584c9b920abe776e43705e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/ca05207bf2b446c6bb45a4f99c07abcf, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f307a165cba44528b2c022ba7a3fb467, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/fde6cc593541404580d3ffb371b1b235, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/ce0dd2e096d54cfb925c4b06e0da9967, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/86bc3c80c06b487ab1055f0a9fa33f45, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/0effa522f53940d38f6d69f2bc1ff80c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/31965c52fb47460281c770b759141047, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/2d8656c95d174d4986962d26bccf4e92, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/3b7fc437769446ccaa53e99cfaa1b268, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b38e705aca114e54bf989e9f3a3b4832, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/1abe0d35b90e4333a825bb96aa8815ef, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b8e2556a47bd4172b2b7585b004cf435, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/1960564528a5498fb71a8be63d7d53b1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b5bc5155545a43cfb173b33b03b3d88b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/7e81d090cb3b4482abec74e13449669a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f1bc94c5f1ee41279b9797d03903e751, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f92862ae76f5498fa0903715afbbb40d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/95f0db1297c14956a5943a07e84a609d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/409f0593853049b6866fa86af8fb86c0] to archive 2023-09-06T20:15:29,866 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:15:29,868 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/954b3a9bb66d4930ac5f3583ac717b3a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/954b3a9bb66d4930ac5f3583ac717b3a 2023-09-06T20:15:29,869 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/bdbe6debf3504c34b1076b790d961106 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/bdbe6debf3504c34b1076b790d961106 2023-09-06T20:15:29,870 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/96e221ae23e5401a8e159ec883dcf452 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/96e221ae23e5401a8e159ec883dcf452 2023-09-06T20:15:29,871 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/0661da3230f34d3faf6047111114679a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/0661da3230f34d3faf6047111114679a 2023-09-06T20:15:29,872 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/5d889ee1955d4b8fa079239a928e6cb7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/5d889ee1955d4b8fa079239a928e6cb7 2023-09-06T20:15:29,873 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f76895f6b7584c9b920abe776e43705e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f76895f6b7584c9b920abe776e43705e 2023-09-06T20:15:29,874 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/ca05207bf2b446c6bb45a4f99c07abcf to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/ca05207bf2b446c6bb45a4f99c07abcf 2023-09-06T20:15:29,875 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f307a165cba44528b2c022ba7a3fb467 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f307a165cba44528b2c022ba7a3fb467 2023-09-06T20:15:29,876 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/fde6cc593541404580d3ffb371b1b235 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/fde6cc593541404580d3ffb371b1b235 2023-09-06T20:15:29,877 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/ce0dd2e096d54cfb925c4b06e0da9967 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/ce0dd2e096d54cfb925c4b06e0da9967 2023-09-06T20:15:29,878 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/86bc3c80c06b487ab1055f0a9fa33f45 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/86bc3c80c06b487ab1055f0a9fa33f45 2023-09-06T20:15:29,879 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/0effa522f53940d38f6d69f2bc1ff80c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/0effa522f53940d38f6d69f2bc1ff80c 2023-09-06T20:15:29,879 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/31965c52fb47460281c770b759141047 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/31965c52fb47460281c770b759141047 2023-09-06T20:15:29,880 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/2d8656c95d174d4986962d26bccf4e92 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/2d8656c95d174d4986962d26bccf4e92 2023-09-06T20:15:29,881 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/3b7fc437769446ccaa53e99cfaa1b268 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/3b7fc437769446ccaa53e99cfaa1b268 2023-09-06T20:15:29,882 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b38e705aca114e54bf989e9f3a3b4832 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b38e705aca114e54bf989e9f3a3b4832 2023-09-06T20:15:29,883 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/1abe0d35b90e4333a825bb96aa8815ef to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/1abe0d35b90e4333a825bb96aa8815ef 2023-09-06T20:15:29,884 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b8e2556a47bd4172b2b7585b004cf435 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b8e2556a47bd4172b2b7585b004cf435 2023-09-06T20:15:29,885 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/1960564528a5498fb71a8be63d7d53b1 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/1960564528a5498fb71a8be63d7d53b1 2023-09-06T20:15:29,886 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b5bc5155545a43cfb173b33b03b3d88b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/b5bc5155545a43cfb173b33b03b3d88b 2023-09-06T20:15:29,887 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/7e81d090cb3b4482abec74e13449669a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/7e81d090cb3b4482abec74e13449669a 2023-09-06T20:15:29,888 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f1bc94c5f1ee41279b9797d03903e751 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f1bc94c5f1ee41279b9797d03903e751 2023-09-06T20:15:29,889 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f92862ae76f5498fa0903715afbbb40d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/f92862ae76f5498fa0903715afbbb40d 2023-09-06T20:15:29,890 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/95f0db1297c14956a5943a07e84a609d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/95f0db1297c14956a5943a07e84a609d 2023-09-06T20:15:29,891 DEBUG [StoreCloser-TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/409f0593853049b6866fa86af8fb86c0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/409f0593853049b6866fa86af8fb86c0 2023-09-06T20:15:29,895 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/recovered.edits/383.seqid, newMaxSeqId=383, maxSeqId=4 2023-09-06T20:15:29,896 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906. 2023-09-06T20:15:29,896 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(1622): Region close journal for 3bb591e2a9ad6f7c9630808320336906: 2023-09-06T20:15:29,897 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] handler.UnassignRegionHandler(163): Closed 3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:29,897 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=126 updating hbase:meta row=3bb591e2a9ad6f7c9630808320336906, regionState=CLOSED 2023-09-06T20:15:29,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=127, resume processing ppid=126 2023-09-06T20:15:29,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=127, ppid=126, state=SUCCESS; CloseRegionProcedure 3bb591e2a9ad6f7c9630808320336906, server=jenkins-hbase3.apache.org,33275,1694031217268 in 1.5140 sec 2023-09-06T20:15:29,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=126, resume processing ppid=125 2023-09-06T20:15:29,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=126, ppid=125, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bb591e2a9ad6f7c9630808320336906, UNASSIGN in 1.5200 sec 2023-09-06T20:15:29,902 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031329901"}]},"ts":"1694031329901"} 2023-09-06T20:15:29,902 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2023-09-06T20:15:29,904 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2023-09-06T20:15:29,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=125, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.5320 sec 2023-09-06T20:15:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=125 2023-09-06T20:15:30,480 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 125 completed 2023-09-06T20:15:30,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete TestAcidGuarantees 2023-09-06T20:15:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=128, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:30,482 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=128, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=128 2023-09-06T20:15:30,483 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=128, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:30,486 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,489 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/recovered.edits] 2023-09-06T20:15:30,491 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/88536388725c4451b4697b8b16ecc335 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/88536388725c4451b4697b8b16ecc335 2023-09-06T20:15:30,492 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/c97d8a74c9a64d2caf1703b9d20d3fc4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/c97d8a74c9a64d2caf1703b9d20d3fc4 2023-09-06T20:15:30,493 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/e188de3d4684491190c4d04286f7db0d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/A/e188de3d4684491190c4d04286f7db0d 2023-09-06T20:15:30,495 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/20ca81cd375a44e2bc6c32a156d8f6e3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/20ca81cd375a44e2bc6c32a156d8f6e3 2023-09-06T20:15:30,496 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/707c130a988c43edae75c5b3dd3e5f05 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/707c130a988c43edae75c5b3dd3e5f05 2023-09-06T20:15:30,497 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/d8a69f90faa64a6697ac657e0079be5b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/B/d8a69f90faa64a6697ac657e0079be5b 2023-09-06T20:15:30,499 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/d30b0a56ebdd464cae488dd077806650 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/d30b0a56ebdd464cae488dd077806650 2023-09-06T20:15:30,501 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/dd9532809bfc459a94da1336419ca898 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/dd9532809bfc459a94da1336419ca898 2023-09-06T20:15:30,502 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/eb4ebdf7f4424e1bb6faaad7ec316584 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/C/eb4ebdf7f4424e1bb6faaad7ec316584 2023-09-06T20:15:30,504 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/recovered.edits/383.seqid to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906/recovered.edits/383.seqid 2023-09-06T20:15:30,504 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,504 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2023-09-06T20:15:30,504 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2023-09-06T20:15:30,505 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2023-09-06T20:15:30,507 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090605b1e2c3b27d4a609de4c403cb64a25d_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090605b1e2c3b27d4a609de4c403cb64a25d_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,508 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309060b4d66dbb5a949fea444a1909837e9c2_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309060b4d66dbb5a949fea444a1909837e9c2_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,509 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090613e19a7d37c14faebc19c36199d1b221_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090613e19a7d37c14faebc19c36199d1b221_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,510 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090614c81fc89c574d08a3c76b92ed401214_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090614c81fc89c574d08a3c76b92ed401214_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,511 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906553e332f3a36400f982f880fbcc34203_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906553e332f3a36400f982f880fbcc34203_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,513 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309065871ad70a7174316b3d67c89282ca145_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309065871ad70a7174316b3d67c89282ca145_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,514 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309065d1932d160344cb7833b2c833d245dd0_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309065d1932d160344cb7833b2c833d245dd0_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,515 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309069d6bd64b595f4de2a59b5b5cfff03f29_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309069d6bd64b595f4de2a59b5b5cfff03f29_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,516 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906a200042fe8c145b883602377ae483f1d_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906a200042fe8c145b883602377ae483f1d_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,517 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906ae89659a58d64ab6aa58146cff8c5acf_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906ae89659a58d64ab6aa58146cff8c5acf_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,518 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c5c89ba9b13a4a2aad6a8550d84362df_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c5c89ba9b13a4a2aad6a8550d84362df_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,518 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c7694ad2f72a43c2a0f0b095d007e3e1_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c7694ad2f72a43c2a0f0b095d007e3e1_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,519 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c7e606cd833f4f08baa585d43be6df8d_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c7e606cd833f4f08baa585d43be6df8d_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,520 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906e69dd326963c4aa1b0efbcea0fdcac28_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906e69dd326963c4aa1b0efbcea0fdcac28_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,521 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906eeb304e70e084ab2bf15ed3c4027493f_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906eeb304e70e084ab2bf15ed3c4027493f_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,522 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f19fb2c5914e4c61b9d1fed4676b603a_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f19fb2c5914e4c61b9d1fed4676b603a_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,523 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f3ca23f3e72c49e189184bae6170b80c_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f3ca23f3e72c49e189184bae6170b80c_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,525 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f65580c7dd94478eb4b14dd60b317ac6_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f65580c7dd94478eb4b14dd60b317ac6_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,526 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f8173b701c6d4017b64899205906ac19_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f8173b701c6d4017b64899205906ac19_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,527 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f8b3fb7b02b34af2a98ce76618717074_3bb591e2a9ad6f7c9630808320336906 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f8b3fb7b02b34af2a98ce76618717074_3bb591e2a9ad6f7c9630808320336906 2023-09-06T20:15:30,527 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2023-09-06T20:15:30,529 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=128, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:30,530 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2023-09-06T20:15:30,532 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2023-09-06T20:15:30,532 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=128, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:30,532 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2023-09-06T20:15:30,533 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694031330532"}]},"ts":"9223372036854775807"} 2023-09-06T20:15:30,534 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:15:30,534 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 3bb591e2a9ad6f7c9630808320336906, NAME => 'TestAcidGuarantees,,1694031304482.3bb591e2a9ad6f7c9630808320336906.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:15:30,534 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2023-09-06T20:15:30,534 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694031330534"}]},"ts":"9223372036854775807"} 2023-09-06T20:15:30,536 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2023-09-06T20:15:30,539 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(127): Finished pid=128, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:30,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=128, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2023-09-06T20:15:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=128 2023-09-06T20:15:30,584 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 128 completed 2023-09-06T20:15:30,598 INFO [Listener at localhost/35961 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testMobScanAtomicity Thread=222 (was 219) - Thread LEAK? -, OpenFileDescriptor=568 (was 599), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=609 (was 509) - SystemLoadAverage LEAK? -, ProcessCount=167 (was 167), AvailableMemoryMB=4878 (was 4953) 2023-09-06T20:15:30,610 INFO [Listener at localhost/35961 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testScanAtomicity Thread=222, OpenFileDescriptor=568, MaxFileDescriptor=60000, SystemLoadAverage=609, ProcessCount=167, AvailableMemoryMB=4877 2023-09-06T20:15:30,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-06T20:15:30,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:15:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=129, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:30,614 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=129, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:15:30,614 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:30,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 129 2023-09-06T20:15:30,615 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=129, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:15:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=129 2023-09-06T20:15:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=129 2023-09-06T20:15:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=129 2023-09-06T20:15:31,022 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236 2023-09-06T20:15:31,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:15:31,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1668): Closing 969b36246b1411b7cf0e9c1debb5278b, disabling compactions & flushes 2023-09-06T20:15:31,028 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:31,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:31,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. after waiting 0 ms 2023-09-06T20:15:31,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:31,028 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:31,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1622): Region close journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:31,029 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=129, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:15:31,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1694031331029"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694031331029"}]},"ts":"1694031331029"} 2023-09-06T20:15:31,030 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:15:31,031 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=129, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:15:31,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031331031"}]},"ts":"1694031331031"} 2023-09-06T20:15:31,031 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2023-09-06T20:15:31,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=969b36246b1411b7cf0e9c1debb5278b, ASSIGN}] 2023-09-06T20:15:31,046 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=130, ppid=129, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=969b36246b1411b7cf0e9c1debb5278b, ASSIGN 2023-09-06T20:15:31,048 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=130, ppid=129, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=969b36246b1411b7cf0e9c1debb5278b, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33275,1694031217268; forceNewPlan=false, retain=false 2023-09-06T20:15:31,199 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=130 updating hbase:meta row=969b36246b1411b7cf0e9c1debb5278b, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:31,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE; OpenRegionProcedure 969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:15:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=129 2023-09-06T20:15:31,354 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:31,354 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7254): Opening region: {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:15:31,354 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:31,354 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:15:31,355 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7296): checking encryption for 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:31,355 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7299): checking classloading for 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:31,355 INFO [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:31,356 INFO [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:31,357 INFO [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 969b36246b1411b7cf0e9c1debb5278b columnFamilyName A 2023-09-06T20:15:31,357 DEBUG [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:31,357 INFO [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] regionserver.HStore(324): Store=969b36246b1411b7cf0e9c1debb5278b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:31,357 INFO [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:31,358 INFO [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:31,358 INFO [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 969b36246b1411b7cf0e9c1debb5278b columnFamilyName B 2023-09-06T20:15:31,358 DEBUG [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:31,359 INFO [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] regionserver.HStore(324): Store=969b36246b1411b7cf0e9c1debb5278b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:31,359 INFO [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:31,360 INFO [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:31,360 INFO [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 969b36246b1411b7cf0e9c1debb5278b columnFamilyName C 2023-09-06T20:15:31,361 DEBUG [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:31,361 INFO [StoreOpener-969b36246b1411b7cf0e9c1debb5278b-1 {}] regionserver.HStore(324): Store=969b36246b1411b7cf0e9c1debb5278b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:31,361 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:31,362 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:31,362 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:31,364 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-06T20:15:31,365 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1076): writing seq id for 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:31,367 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:15:31,367 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1093): Opened 969b36246b1411b7cf0e9c1debb5278b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60546751, jitterRate=-0.09778310358524323}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-06T20:15:31,368 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(992): Region open journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:31,369 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., pid=131, masterSystemTime=1694031331351 2023-09-06T20:15:31,370 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:31,370 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:31,370 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=130 updating hbase:meta row=969b36246b1411b7cf0e9c1debb5278b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:31,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=131, resume processing ppid=130 2023-09-06T20:15:31,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=131, ppid=130, state=SUCCESS; OpenRegionProcedure 969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 in 171 msec 2023-09-06T20:15:31,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=130, resume processing ppid=129 2023-09-06T20:15:31,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=130, ppid=129, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=969b36246b1411b7cf0e9c1debb5278b, ASSIGN in 328 msec 2023-09-06T20:15:31,374 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=129, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:15:31,375 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031331374"}]},"ts":"1694031331374"} 2023-09-06T20:15:31,375 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2023-09-06T20:15:31,378 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=129, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:15:31,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=129, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 766 msec 2023-09-06T20:15:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=129 2023-09-06T20:15:31,719 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 129 completed 2023-09-06T20:15:31,719 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x287d13f5 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:31,725 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffb41e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:31,726 DEBUG [Listener at localhost/35961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:31,728 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42940, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:31,729 DEBUG [Listener at localhost/35961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:15:31,730 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34130, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:15:31,732 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x05908814 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:31,736 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b2d1e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:31,737 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x768de042 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:31,742 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77dc16af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:31,743 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4f8d201b to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:31,748 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@561b5658, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:31,748 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x70c70c97 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:31,752 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@566c776c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:31,752 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x319d6df7 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:31,756 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79766ce7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:31,756 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x06f6e6dc to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:31,760 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cada3b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:31,761 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0f6aeb1b to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:31,769 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e1b3c2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:31,770 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x560b4319 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:31,774 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ee148f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:31,775 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x233f731e to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:31,780 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f20efeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:31,780 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x258a8814 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:31,784 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7df5a838, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:31,789 DEBUG [hconnection-0x6edeae5d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:31,789 DEBUG [hconnection-0x436bc692-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:31,790 DEBUG [hconnection-0x3d5edab3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:31,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:31,790 DEBUG [hconnection-0x33aadd4e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:31,791 DEBUG [hconnection-0x7eb1c5ba-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:31,791 DEBUG [hconnection-0x38f1f834-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees 2023-09-06T20:15:31,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=132 2023-09-06T20:15:31,794 DEBUG [hconnection-0x69fdf8ec-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:31,795 DEBUG [hconnection-0x2d572ca5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:31,795 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42944, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:31,795 DEBUG [hconnection-0xf5685f3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:31,795 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42976, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:31,796 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42990, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:31,798 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:31,798 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42948, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:31,798 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42964, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:31,798 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42980, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:31,798 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42984, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:31,798 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43004, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:31,799 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43018, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:31,799 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=132, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:31,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:31,801 DEBUG [hconnection-0x707df626-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:31,803 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43028, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:31,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:15:31,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:31,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:31,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:31,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:31,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:31,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:31,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/ffacdf7c464346b38a66eddb6017ee74 is 50, key is test_row_0/A:col10/1694031331812/Put/seqid=0 2023-09-06T20:15:31,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031391832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:31,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:31,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031391833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:31,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031391839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:31,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:31,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031391842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:31,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:31,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031391842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:31,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/ffacdf7c464346b38a66eddb6017ee74 2023-09-06T20:15:31,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/36329a47593e462fbf092f3bef1b481c is 50, key is test_row_0/B:col10/1694031331812/Put/seqid=0 2023-09-06T20:15:31,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/36329a47593e462fbf092f3bef1b481c 2023-09-06T20:15:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=132 2023-09-06T20:15:31,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/834244a735224e6fa2bacbd1291f29b9 is 50, key is test_row_0/C:col10/1694031331812/Put/seqid=0 2023-09-06T20:15:31,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:31,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:31,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031391940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:31,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031391940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:31,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:31,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031391946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:31,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031391950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:31,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031391950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:31,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2023-09-06T20:15:31,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:31,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:31,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:31,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(53): pid=133 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:31,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=132 2023-09-06T20:15:32,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2023-09-06T20:15:32,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:32,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:32,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:32,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(53): pid=133 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:32,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:32,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:32,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031392143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031392144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031392148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031392153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031392154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2023-09-06T20:15:32,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:32,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:32,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:32,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(53): pid=133 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:32,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:32,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/834244a735224e6fa2bacbd1291f29b9 2023-09-06T20:15:32,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/ffacdf7c464346b38a66eddb6017ee74 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/ffacdf7c464346b38a66eddb6017ee74 2023-09-06T20:15:32,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/ffacdf7c464346b38a66eddb6017ee74, entries=250, sequenceid=16, filesize=16.3 K 2023-09-06T20:15:32,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/36329a47593e462fbf092f3bef1b481c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/36329a47593e462fbf092f3bef1b481c 2023-09-06T20:15:32,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/36329a47593e462fbf092f3bef1b481c, entries=150, sequenceid=16, filesize=11.7 K 2023-09-06T20:15:32,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/834244a735224e6fa2bacbd1291f29b9 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/834244a735224e6fa2bacbd1291f29b9 2023-09-06T20:15:32,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/834244a735224e6fa2bacbd1291f29b9, entries=150, sequenceid=16, filesize=11.7 K 2023-09-06T20:15:32,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 969b36246b1411b7cf0e9c1debb5278b in 529ms, sequenceid=16, compaction requested=false 2023-09-06T20:15:32,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=132 2023-09-06T20:15:32,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2023-09-06T20:15:32,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:32,415 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2023-09-06T20:15:32,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:32,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:32,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:32,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:32,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:32,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:32,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/e9dc51eaa3794c6da7e736ce9e20234f is 50, key is test_row_0/A:col10/1694031331831/Put/seqid=0 2023-09-06T20:15:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:32,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:32,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031392461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031392463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031392463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031392464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031392465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031392566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031392567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031392569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031392569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031392569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031392770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031392772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031392772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031392774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031392775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:32,852 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/e9dc51eaa3794c6da7e736ce9e20234f 2023-09-06T20:15:32,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/8f3775be6c604eada3a253f8d04c645b is 50, key is test_row_0/B:col10/1694031331831/Put/seqid=0 2023-09-06T20:15:32,864 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/8f3775be6c604eada3a253f8d04c645b 2023-09-06T20:15:32,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/a16fdff61596487d97192e05c60887a4 is 50, key is test_row_0/C:col10/1694031331831/Put/seqid=0 2023-09-06T20:15:32,885 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/a16fdff61596487d97192e05c60887a4 2023-09-06T20:15:32,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/e9dc51eaa3794c6da7e736ce9e20234f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e9dc51eaa3794c6da7e736ce9e20234f 2023-09-06T20:15:32,893 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e9dc51eaa3794c6da7e736ce9e20234f, entries=150, sequenceid=37, filesize=11.7 K 2023-09-06T20:15:32,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/8f3775be6c604eada3a253f8d04c645b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8f3775be6c604eada3a253f8d04c645b 2023-09-06T20:15:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=132 2023-09-06T20:15:32,899 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8f3775be6c604eada3a253f8d04c645b, entries=150, sequenceid=37, filesize=11.7 K 2023-09-06T20:15:32,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/a16fdff61596487d97192e05c60887a4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a16fdff61596487d97192e05c60887a4 2023-09-06T20:15:32,903 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a16fdff61596487d97192e05c60887a4, entries=150, sequenceid=37, filesize=11.7 K 2023-09-06T20:15:32,904 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(3022): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 969b36246b1411b7cf0e9c1debb5278b in 489ms, sequenceid=37, compaction requested=false 2023-09-06T20:15:32,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:32,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:32,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=133 2023-09-06T20:15:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=133 2023-09-06T20:15:32,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=133, resume processing ppid=132 2023-09-06T20:15:32,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=133, ppid=132, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1060 sec 2023-09-06T20:15:32,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=132, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees in 1.1180 sec 2023-09-06T20:15:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:33,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2023-09-06T20:15:33,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:33,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:33,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:33,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:33,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:33,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:33,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/462c4428bbb74f0db0ad093edc0cbe52 is 50, key is test_row_0/A:col10/1694031333075/Put/seqid=0 2023-09-06T20:15:33,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031393126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031393127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031393127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031393128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031393129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031393233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031393233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031393234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031393235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031393235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031393439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031393439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031393439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031393440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031393441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/462c4428bbb74f0db0ad093edc0cbe52 2023-09-06T20:15:33,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/83a068dbb52149b884e1d87c8972bf14 is 50, key is test_row_0/B:col10/1694031333075/Put/seqid=0 2023-09-06T20:15:33,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031393744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031393745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031393745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031393745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031393745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=132 2023-09-06T20:15:33,897 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 132 completed 2023-09-06T20:15:33,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees 2023-09-06T20:15:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=134 2023-09-06T20:15:33,901 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:33,901 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=134, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:33,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:33,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/83a068dbb52149b884e1d87c8972bf14 2023-09-06T20:15:33,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/fe9a997c83be4d7b8c70dddced91f1d0 is 50, key is test_row_0/C:col10/1694031333075/Put/seqid=0 2023-09-06T20:15:33,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/fe9a997c83be4d7b8c70dddced91f1d0 2023-09-06T20:15:33,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/462c4428bbb74f0db0ad093edc0cbe52 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/462c4428bbb74f0db0ad093edc0cbe52 2023-09-06T20:15:33,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/462c4428bbb74f0db0ad093edc0cbe52, entries=150, sequenceid=53, filesize=11.7 K 2023-09-06T20:15:33,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/83a068dbb52149b884e1d87c8972bf14 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/83a068dbb52149b884e1d87c8972bf14 2023-09-06T20:15:33,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/83a068dbb52149b884e1d87c8972bf14, entries=150, sequenceid=53, filesize=11.7 K 2023-09-06T20:15:33,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/fe9a997c83be4d7b8c70dddced91f1d0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/fe9a997c83be4d7b8c70dddced91f1d0 2023-09-06T20:15:33,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/fe9a997c83be4d7b8c70dddced91f1d0, entries=150, sequenceid=53, filesize=11.7 K 2023-09-06T20:15:33,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 969b36246b1411b7cf0e9c1debb5278b in 874ms, sequenceid=53, compaction requested=true 2023-09-06T20:15:33,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:33,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:33,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:33,951 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:33,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:33,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:33,951 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:33,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:33,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:33,953 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40632 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:33,953 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/A is initiating minor compaction (all files) 2023-09-06T20:15:33,953 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/A in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:33,953 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:33,953 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/B is initiating minor compaction (all files) 2023-09-06T20:15:33,953 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/ffacdf7c464346b38a66eddb6017ee74, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e9dc51eaa3794c6da7e736ce9e20234f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/462c4428bbb74f0db0ad093edc0cbe52] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=39.7 K 2023-09-06T20:15:33,953 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/B in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:33,954 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/36329a47593e462fbf092f3bef1b481c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8f3775be6c604eada3a253f8d04c645b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/83a068dbb52149b884e1d87c8972bf14] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=35.1 K 2023-09-06T20:15:33,954 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting ffacdf7c464346b38a66eddb6017ee74, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1694031331812 2023-09-06T20:15:33,954 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 36329a47593e462fbf092f3bef1b481c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1694031331812 2023-09-06T20:15:33,955 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting e9dc51eaa3794c6da7e736ce9e20234f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1694031331831 2023-09-06T20:15:33,955 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 8f3775be6c604eada3a253f8d04c645b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1694031331831 2023-09-06T20:15:33,955 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 462c4428bbb74f0db0ad093edc0cbe52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1694031332459 2023-09-06T20:15:33,956 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 83a068dbb52149b884e1d87c8972bf14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1694031332459 2023-09-06T20:15:33,967 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#A#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:33,968 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/afa13eca4005415bb205ec70c02929a6 is 50, key is test_row_0/A:col10/1694031333075/Put/seqid=0 2023-09-06T20:15:33,975 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#B#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:33,976 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/6f50d857313940d1ad599e1b6e1d7def is 50, key is test_row_0/B:col10/1694031333075/Put/seqid=0 2023-09-06T20:15:34,002 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/afa13eca4005415bb205ec70c02929a6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/afa13eca4005415bb205ec70c02929a6 2023-09-06T20:15:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=134 2023-09-06T20:15:34,011 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/A of 969b36246b1411b7cf0e9c1debb5278b into afa13eca4005415bb205ec70c02929a6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:34,011 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:34,011 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/A, priority=13, startTime=1694031333951; duration=0sec 2023-09-06T20:15:34,011 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:34,011 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:A 2023-09-06T20:15:34,011 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:34,012 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:34,012 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/C is initiating minor compaction (all files) 2023-09-06T20:15:34,012 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/C in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:34,012 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/834244a735224e6fa2bacbd1291f29b9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a16fdff61596487d97192e05c60887a4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/fe9a997c83be4d7b8c70dddced91f1d0] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=35.1 K 2023-09-06T20:15:34,013 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 834244a735224e6fa2bacbd1291f29b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1694031331812 2023-09-06T20:15:34,013 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/6f50d857313940d1ad599e1b6e1d7def as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/6f50d857313940d1ad599e1b6e1d7def 2023-09-06T20:15:34,014 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting a16fdff61596487d97192e05c60887a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1694031331831 2023-09-06T20:15:34,015 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting fe9a997c83be4d7b8c70dddced91f1d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1694031332459 2023-09-06T20:15:34,018 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/B of 969b36246b1411b7cf0e9c1debb5278b into 6f50d857313940d1ad599e1b6e1d7def(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:34,018 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:34,018 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/B, priority=13, startTime=1694031333951; duration=0sec 2023-09-06T20:15:34,018 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:34,018 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:B 2023-09-06T20:15:34,023 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#C#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:34,024 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/a32dc0695c0045e882fcb317f0f1b3b6 is 50, key is test_row_0/C:col10/1694031333075/Put/seqid=0 2023-09-06T20:15:34,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2023-09-06T20:15:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:34,055 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2023-09-06T20:15:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:34,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/5902983c674b4116b37d9becf7722b21 is 50, key is test_row_0/A:col10/1694031333126/Put/seqid=0 2023-09-06T20:15:34,082 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/5902983c674b4116b37d9becf7722b21 2023-09-06T20:15:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/658c09e67d0a47b8bcadca37245dd9de is 50, key is test_row_0/B:col10/1694031333126/Put/seqid=0 2023-09-06T20:15:34,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=134 2023-09-06T20:15:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:34,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:34,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031394261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031394263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031394266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031394267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031394268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031394369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031394371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031394373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031394373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031394373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,436 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/a32dc0695c0045e882fcb317f0f1b3b6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a32dc0695c0045e882fcb317f0f1b3b6 2023-09-06T20:15:34,439 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/C of 969b36246b1411b7cf0e9c1debb5278b into a32dc0695c0045e882fcb317f0f1b3b6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:34,439 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:34,439 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/C, priority=13, startTime=1694031333951; duration=0sec 2023-09-06T20:15:34,439 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:34,439 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:C 2023-09-06T20:15:34,491 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/658c09e67d0a47b8bcadca37245dd9de 2023-09-06T20:15:34,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/9986b3b934af4c67ae393183e0bc8d86 is 50, key is test_row_0/C:col10/1694031333126/Put/seqid=0 2023-09-06T20:15:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=134 2023-09-06T20:15:34,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031394577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031394577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031394577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031394577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031394577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031394883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031394883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031394883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031394883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031394883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:34,916 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/9986b3b934af4c67ae393183e0bc8d86 2023-09-06T20:15:34,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/5902983c674b4116b37d9becf7722b21 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/5902983c674b4116b37d9becf7722b21 2023-09-06T20:15:34,925 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/5902983c674b4116b37d9becf7722b21, entries=150, sequenceid=76, filesize=11.7 K 2023-09-06T20:15:34,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/658c09e67d0a47b8bcadca37245dd9de as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/658c09e67d0a47b8bcadca37245dd9de 2023-09-06T20:15:34,930 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/658c09e67d0a47b8bcadca37245dd9de, entries=150, sequenceid=76, filesize=11.7 K 2023-09-06T20:15:34,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/9986b3b934af4c67ae393183e0bc8d86 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/9986b3b934af4c67ae393183e0bc8d86 2023-09-06T20:15:34,935 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/9986b3b934af4c67ae393183e0bc8d86, entries=150, sequenceid=76, filesize=11.7 K 2023-09-06T20:15:34,936 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(3022): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 969b36246b1411b7cf0e9c1debb5278b in 881ms, sequenceid=76, compaction requested=false 2023-09-06T20:15:34,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:34,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:34,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=135 2023-09-06T20:15:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=135 2023-09-06T20:15:34,938 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=135, resume processing ppid=134 2023-09-06T20:15:34,938 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=135, ppid=134, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0350 sec 2023-09-06T20:15:34,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=134, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees in 1.0410 sec 2023-09-06T20:15:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=134 2023-09-06T20:15:35,005 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 134 completed 2023-09-06T20:15:35,007 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees 2023-09-06T20:15:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=136 2023-09-06T20:15:35,009 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:35,010 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=136, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:35,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=136 2023-09-06T20:15:35,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2023-09-06T20:15:35,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:35,163 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-06T20:15:35,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:35,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:35,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:35,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:35,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:35,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:35,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/9e75067f938a4078a34a84e960281086 is 50, key is test_row_0/A:col10/1694031334259/Put/seqid=0 2023-09-06T20:15:35,183 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/9e75067f938a4078a34a84e960281086 2023-09-06T20:15:35,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/03cb71a1225a444b9fc4e4260fda29c3 is 50, key is test_row_0/B:col10/1694031334259/Put/seqid=0 2023-09-06T20:15:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=136 2023-09-06T20:15:35,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:35,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031395407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031395408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031395412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031395412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031395413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,426 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-06T20:15:35,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031395514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031395514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031395517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031395517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031395519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,603 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/03cb71a1225a444b9fc4e4260fda29c3 2023-09-06T20:15:35,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/349326de8d024c9684891754857073e3 is 50, key is test_row_0/C:col10/1694031334259/Put/seqid=0 2023-09-06T20:15:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=136 2023-09-06T20:15:35,614 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/349326de8d024c9684891754857073e3 2023-09-06T20:15:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/9e75067f938a4078a34a84e960281086 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/9e75067f938a4078a34a84e960281086 2023-09-06T20:15:35,622 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/9e75067f938a4078a34a84e960281086, entries=150, sequenceid=92, filesize=11.7 K 2023-09-06T20:15:35,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/03cb71a1225a444b9fc4e4260fda29c3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/03cb71a1225a444b9fc4e4260fda29c3 2023-09-06T20:15:35,626 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/03cb71a1225a444b9fc4e4260fda29c3, entries=150, sequenceid=92, filesize=11.7 K 2023-09-06T20:15:35,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/349326de8d024c9684891754857073e3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/349326de8d024c9684891754857073e3 2023-09-06T20:15:35,631 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/349326de8d024c9684891754857073e3, entries=150, sequenceid=92, filesize=11.7 K 2023-09-06T20:15:35,631 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 969b36246b1411b7cf0e9c1debb5278b in 468ms, sequenceid=92, compaction requested=true 2023-09-06T20:15:35,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:35,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:35,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=137 2023-09-06T20:15:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=137 2023-09-06T20:15:35,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=137, resume processing ppid=136 2023-09-06T20:15:35,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=137, ppid=136, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 622 msec 2023-09-06T20:15:35,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=136, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees in 627 msec 2023-09-06T20:15:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:35,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2023-09-06T20:15:35,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:35,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:35,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:35,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:35,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:35,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:35,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/231cfa8a337d4758b0ecd27bbbd9bf4b is 50, key is test_row_0/A:col10/1694031335412/Put/seqid=0 2023-09-06T20:15:35,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/231cfa8a337d4758b0ecd27bbbd9bf4b 2023-09-06T20:15:35,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/966836b14601421ab8ea6554d1c5057e is 50, key is test_row_0/B:col10/1694031335412/Put/seqid=0 2023-09-06T20:15:35,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031395734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031395735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031395736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031395736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031395737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/966836b14601421ab8ea6554d1c5057e 2023-09-06T20:15:35,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/17cd08f7fb164e21b1028544f7d1922c is 50, key is test_row_0/C:col10/1694031335412/Put/seqid=0 2023-09-06T20:15:35,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031395844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031395844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031395844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031395844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:35,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031395846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031396051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031396052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031396052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031396052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031396053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=136 2023-09-06T20:15:36,113 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 136 completed 2023-09-06T20:15:36,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2023-09-06T20:15:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=138 2023-09-06T20:15:36,118 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:36,118 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:36,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:36,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/17cd08f7fb164e21b1028544f7d1922c 2023-09-06T20:15:36,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/231cfa8a337d4758b0ecd27bbbd9bf4b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/231cfa8a337d4758b0ecd27bbbd9bf4b 2023-09-06T20:15:36,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/231cfa8a337d4758b0ecd27bbbd9bf4b, entries=200, sequenceid=114, filesize=14.0 K 2023-09-06T20:15:36,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/966836b14601421ab8ea6554d1c5057e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/966836b14601421ab8ea6554d1c5057e 2023-09-06T20:15:36,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/966836b14601421ab8ea6554d1c5057e, entries=150, sequenceid=114, filesize=11.7 K 2023-09-06T20:15:36,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/17cd08f7fb164e21b1028544f7d1922c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/17cd08f7fb164e21b1028544f7d1922c 2023-09-06T20:15:36,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/17cd08f7fb164e21b1028544f7d1922c, entries=150, sequenceid=114, filesize=11.7 K 2023-09-06T20:15:36,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 969b36246b1411b7cf0e9c1debb5278b in 485ms, sequenceid=114, compaction requested=true 2023-09-06T20:15:36,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:36,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:36,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:36,208 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:36,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:36,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:36,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:36,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-06T20:15:36,208 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:36,210 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48039 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:36,210 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/B is initiating minor compaction (all files) 2023-09-06T20:15:36,211 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/B in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:36,211 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/6f50d857313940d1ad599e1b6e1d7def, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/658c09e67d0a47b8bcadca37245dd9de, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/03cb71a1225a444b9fc4e4260fda29c3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/966836b14601421ab8ea6554d1c5057e] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=46.9 K 2023-09-06T20:15:36,211 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50379 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:36,211 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/A is initiating minor compaction (all files) 2023-09-06T20:15:36,211 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/A in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:36,211 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/afa13eca4005415bb205ec70c02929a6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/5902983c674b4116b37d9becf7722b21, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/9e75067f938a4078a34a84e960281086, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/231cfa8a337d4758b0ecd27bbbd9bf4b] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=49.2 K 2023-09-06T20:15:36,211 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 6f50d857313940d1ad599e1b6e1d7def, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1694031332459 2023-09-06T20:15:36,211 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting afa13eca4005415bb205ec70c02929a6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1694031332459 2023-09-06T20:15:36,212 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 658c09e67d0a47b8bcadca37245dd9de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1694031333114 2023-09-06T20:15:36,212 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5902983c674b4116b37d9becf7722b21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1694031333114 2023-09-06T20:15:36,212 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 03cb71a1225a444b9fc4e4260fda29c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1694031334259 2023-09-06T20:15:36,213 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 966836b14601421ab8ea6554d1c5057e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1694031335406 2023-09-06T20:15:36,213 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9e75067f938a4078a34a84e960281086, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1694031334259 2023-09-06T20:15:36,214 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 231cfa8a337d4758b0ecd27bbbd9bf4b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1694031335405 2023-09-06T20:15:36,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=138 2023-09-06T20:15:36,227 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#B#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:36,228 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/97131f0636af4001973473b63d869c1c is 50, key is test_row_0/B:col10/1694031335412/Put/seqid=0 2023-09-06T20:15:36,236 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#A#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:36,236 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/3d22221596a24702a46675cc1b05f7cd is 50, key is test_row_0/A:col10/1694031335412/Put/seqid=0 2023-09-06T20:15:36,262 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/97131f0636af4001973473b63d869c1c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/97131f0636af4001973473b63d869c1c 2023-09-06T20:15:36,267 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/B of 969b36246b1411b7cf0e9c1debb5278b into 97131f0636af4001973473b63d869c1c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:36,267 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:36,267 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/B, priority=12, startTime=1694031336208; duration=0sec 2023-09-06T20:15:36,267 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:36,267 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:B 2023-09-06T20:15:36,267 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:36,267 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/3d22221596a24702a46675cc1b05f7cd as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/3d22221596a24702a46675cc1b05f7cd 2023-09-06T20:15:36,268 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48039 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:36,269 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/C is initiating minor compaction (all files) 2023-09-06T20:15:36,269 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/C in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:36,269 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a32dc0695c0045e882fcb317f0f1b3b6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/9986b3b934af4c67ae393183e0bc8d86, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/349326de8d024c9684891754857073e3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/17cd08f7fb164e21b1028544f7d1922c] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=46.9 K 2023-09-06T20:15:36,269 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting a32dc0695c0045e882fcb317f0f1b3b6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1694031332459 2023-09-06T20:15:36,269 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 9986b3b934af4c67ae393183e0bc8d86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1694031333114 2023-09-06T20:15:36,271 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 349326de8d024c9684891754857073e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1694031334259 2023-09-06T20:15:36,271 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 17cd08f7fb164e21b1028544f7d1922c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1694031335406 2023-09-06T20:15:36,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2023-09-06T20:15:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:36,272 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2023-09-06T20:15:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:36,279 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/A of 969b36246b1411b7cf0e9c1debb5278b into 3d22221596a24702a46675cc1b05f7cd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:36,279 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:36,279 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/A, priority=12, startTime=1694031336208; duration=0sec 2023-09-06T20:15:36,280 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:36,280 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:A 2023-09-06T20:15:36,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/bfe9e92664f64bbe9739e7d274492958 is 50, key is test_row_0/A:col10/1694031335735/Put/seqid=0 2023-09-06T20:15:36,283 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#C#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:36,283 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/472abb726fa643c3a295a43052fd3804 is 50, key is test_row_0/C:col10/1694031335412/Put/seqid=0 2023-09-06T20:15:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:36,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:36,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031396379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031396381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031396386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031396386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031396391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=138 2023-09-06T20:15:36,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031396488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031396491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031396492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031396493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031396497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,687 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/bfe9e92664f64bbe9739e7d274492958 2023-09-06T20:15:36,693 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/472abb726fa643c3a295a43052fd3804 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/472abb726fa643c3a295a43052fd3804 2023-09-06T20:15:36,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/8010908f08594ab5966afe66188477c4 is 50, key is test_row_0/B:col10/1694031335735/Put/seqid=0 2023-09-06T20:15:36,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031396692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031396694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,698 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/C of 969b36246b1411b7cf0e9c1debb5278b into 472abb726fa643c3a295a43052fd3804(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:36,698 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:36,698 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/C, priority=12, startTime=1694031336208; duration=0sec 2023-09-06T20:15:36,698 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:36,699 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:C 2023-09-06T20:15:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031396698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031396698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031396700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=138 2023-09-06T20:15:36,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:36,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031396997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031397000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031397003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031397005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031397005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,104 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/8010908f08594ab5966afe66188477c4 2023-09-06T20:15:37,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/9bab74e5f2654f9bacba8e1db1528608 is 50, key is test_row_0/C:col10/1694031335735/Put/seqid=0 2023-09-06T20:15:37,119 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/9bab74e5f2654f9bacba8e1db1528608 2023-09-06T20:15:37,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/bfe9e92664f64bbe9739e7d274492958 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bfe9e92664f64bbe9739e7d274492958 2023-09-06T20:15:37,129 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bfe9e92664f64bbe9739e7d274492958, entries=150, sequenceid=129, filesize=11.7 K 2023-09-06T20:15:37,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/8010908f08594ab5966afe66188477c4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8010908f08594ab5966afe66188477c4 2023-09-06T20:15:37,133 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8010908f08594ab5966afe66188477c4, entries=150, sequenceid=129, filesize=11.7 K 2023-09-06T20:15:37,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/9bab74e5f2654f9bacba8e1db1528608 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/9bab74e5f2654f9bacba8e1db1528608 2023-09-06T20:15:37,138 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/9bab74e5f2654f9bacba8e1db1528608, entries=150, sequenceid=129, filesize=11.7 K 2023-09-06T20:15:37,139 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3022): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 969b36246b1411b7cf0e9c1debb5278b in 867ms, sequenceid=129, compaction requested=false 2023-09-06T20:15:37,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:37,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:37,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=139 2023-09-06T20:15:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=139 2023-09-06T20:15:37,142 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=139, resume processing ppid=138 2023-09-06T20:15:37,142 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=139, ppid=138, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0230 sec 2023-09-06T20:15:37,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=138, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 1.0280 sec 2023-09-06T20:15:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=138 2023-09-06T20:15:37,220 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 138 completed 2023-09-06T20:15:37,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2023-09-06T20:15:37,224 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:37,225 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:37,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=140 2023-09-06T20:15:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=140 2023-09-06T20:15:37,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2023-09-06T20:15:37,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:37,378 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-06T20:15:37,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:37,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:37,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:37,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:37,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:37,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/d41fec7eeeca4bd0af2b7eee0e9a1863 is 50, key is test_row_0/A:col10/1694031336385/Put/seqid=0 2023-09-06T20:15:37,400 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/d41fec7eeeca4bd0af2b7eee0e9a1863 2023-09-06T20:15:37,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/7a133b1a513e481097ba13bef642a74d is 50, key is test_row_0/B:col10/1694031336385/Put/seqid=0 2023-09-06T20:15:37,411 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/7a133b1a513e481097ba13bef642a74d 2023-09-06T20:15:37,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5579bc46ccc141da803e6e4322392a87 is 50, key is test_row_0/C:col10/1694031336385/Put/seqid=0 2023-09-06T20:15:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:37,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:37,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031397520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031397521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031397521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031397522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031397522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=140 2023-09-06T20:15:37,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031397626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031397626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031397627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031397627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031397628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=140 2023-09-06T20:15:37,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031397830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,835 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5579bc46ccc141da803e6e4322392a87 2023-09-06T20:15:37,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/d41fec7eeeca4bd0af2b7eee0e9a1863 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/d41fec7eeeca4bd0af2b7eee0e9a1863 2023-09-06T20:15:37,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031397834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031397834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031397835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031397836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:37,844 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/d41fec7eeeca4bd0af2b7eee0e9a1863, entries=150, sequenceid=154, filesize=11.8 K 2023-09-06T20:15:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/7a133b1a513e481097ba13bef642a74d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/7a133b1a513e481097ba13bef642a74d 2023-09-06T20:15:37,849 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/7a133b1a513e481097ba13bef642a74d, entries=150, sequenceid=154, filesize=11.8 K 2023-09-06T20:15:37,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5579bc46ccc141da803e6e4322392a87 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5579bc46ccc141da803e6e4322392a87 2023-09-06T20:15:37,853 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5579bc46ccc141da803e6e4322392a87, entries=150, sequenceid=154, filesize=11.8 K 2023-09-06T20:15:37,853 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for 969b36246b1411b7cf0e9c1debb5278b in 475ms, sequenceid=154, compaction requested=true 2023-09-06T20:15:37,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:37,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:37,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=141 2023-09-06T20:15:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=141 2023-09-06T20:15:37,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=141, resume processing ppid=140 2023-09-06T20:15:37,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=141, ppid=140, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 629 msec 2023-09-06T20:15:37,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=140, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 635 msec 2023-09-06T20:15:38,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:38,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2023-09-06T20:15:38,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:38,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:38,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:38,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:38,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:38,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:38,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/729bc5b390664aa2818a60358810253a is 50, key is test_row_0/A:col10/1694031337521/Put/seqid=0 2023-09-06T20:15:38,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031398164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031398164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031398165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031398165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031398167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031398272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031398272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031398273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031398273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031398273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=140 2023-09-06T20:15:38,333 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 140 completed 2023-09-06T20:15:38,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2023-09-06T20:15:38,337 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-06T20:15:38,338 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:38,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-06T20:15:38,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031398478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031398479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031398479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031398480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031398480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2023-09-06T20:15:38,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:38,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:38,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:38,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(53): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:38,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:38,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/729bc5b390664aa2818a60358810253a 2023-09-06T20:15:38,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/2fa43439abd6495cbea030f351904069 is 50, key is test_row_0/B:col10/1694031337521/Put/seqid=0 2023-09-06T20:15:38,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/2fa43439abd6495cbea030f351904069 2023-09-06T20:15:38,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/6b75d3478c8c4cbd9c0633103b24af6a is 50, key is test_row_0/C:col10/1694031337521/Put/seqid=0 2023-09-06T20:15:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-06T20:15:38,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2023-09-06T20:15:38,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:38,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:38,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:38,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(53): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:38,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:38,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031398782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031398783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031398784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031398784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031398785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:38,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2023-09-06T20:15:38,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:38,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:38,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:38,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(53): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:38,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-06T20:15:38,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2023-09-06T20:15:38,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:38,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:38,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:38,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(53): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:38,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:38,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/6b75d3478c8c4cbd9c0633103b24af6a 2023-09-06T20:15:38,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/729bc5b390664aa2818a60358810253a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/729bc5b390664aa2818a60358810253a 2023-09-06T20:15:38,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/729bc5b390664aa2818a60358810253a, entries=200, sequenceid=170, filesize=14.2 K 2023-09-06T20:15:38,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/2fa43439abd6495cbea030f351904069 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/2fa43439abd6495cbea030f351904069 2023-09-06T20:15:38,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/2fa43439abd6495cbea030f351904069, entries=150, sequenceid=170, filesize=11.8 K 2023-09-06T20:15:38,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/6b75d3478c8c4cbd9c0633103b24af6a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/6b75d3478c8c4cbd9c0633103b24af6a 2023-09-06T20:15:38,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/6b75d3478c8c4cbd9c0633103b24af6a, entries=150, sequenceid=170, filesize=11.8 K 2023-09-06T20:15:38,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 969b36246b1411b7cf0e9c1debb5278b in 856ms, sequenceid=170, compaction requested=true 2023-09-06T20:15:38,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:38,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:38,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:38,996 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:38,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:38,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:38,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:38,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:38,996 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:38,997 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50866 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:38,997 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/A is initiating minor compaction (all files) 2023-09-06T20:15:38,997 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/A in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:38,998 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/3d22221596a24702a46675cc1b05f7cd, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bfe9e92664f64bbe9739e7d274492958, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/d41fec7eeeca4bd0af2b7eee0e9a1863, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/729bc5b390664aa2818a60358810253a] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=49.7 K 2023-09-06T20:15:38,998 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:38,998 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3d22221596a24702a46675cc1b05f7cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1694031335406 2023-09-06T20:15:38,998 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/B is initiating minor compaction (all files) 2023-09-06T20:15:38,998 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/B in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:38,998 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/97131f0636af4001973473b63d869c1c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8010908f08594ab5966afe66188477c4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/7a133b1a513e481097ba13bef642a74d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/2fa43439abd6495cbea030f351904069] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=47.3 K 2023-09-06T20:15:38,998 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 97131f0636af4001973473b63d869c1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1694031335406 2023-09-06T20:15:38,998 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting bfe9e92664f64bbe9739e7d274492958, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1694031335733 2023-09-06T20:15:38,999 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 8010908f08594ab5966afe66188477c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1694031335733 2023-09-06T20:15:38,999 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting d41fec7eeeca4bd0af2b7eee0e9a1863, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1694031336377 2023-09-06T20:15:38,999 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 7a133b1a513e481097ba13bef642a74d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1694031336377 2023-09-06T20:15:38,999 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 729bc5b390664aa2818a60358810253a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1694031337519 2023-09-06T20:15:39,000 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 2fa43439abd6495cbea030f351904069, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1694031337520 2023-09-06T20:15:39,008 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#A#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:39,008 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#B#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:39,008 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/34edbd77ac6142dd920468d9e94af21a is 50, key is test_row_0/A:col10/1694031337521/Put/seqid=0 2023-09-06T20:15:39,009 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/eda06905d26f4fdda6348f321e0c08c7 is 50, key is test_row_0/B:col10/1694031337521/Put/seqid=0 2023-09-06T20:15:39,034 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/eda06905d26f4fdda6348f321e0c08c7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/eda06905d26f4fdda6348f321e0c08c7 2023-09-06T20:15:39,038 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/B of 969b36246b1411b7cf0e9c1debb5278b into eda06905d26f4fdda6348f321e0c08c7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:39,038 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:39,038 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/B, priority=12, startTime=1694031338996; duration=0sec 2023-09-06T20:15:39,039 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:39,039 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:B 2023-09-06T20:15:39,039 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:39,040 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:39,040 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/C is initiating minor compaction (all files) 2023-09-06T20:15:39,040 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/C in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:39,040 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/472abb726fa643c3a295a43052fd3804, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/9bab74e5f2654f9bacba8e1db1528608, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5579bc46ccc141da803e6e4322392a87, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/6b75d3478c8c4cbd9c0633103b24af6a] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=47.3 K 2023-09-06T20:15:39,040 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 472abb726fa643c3a295a43052fd3804, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1694031335406 2023-09-06T20:15:39,040 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 9bab74e5f2654f9bacba8e1db1528608, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1694031335733 2023-09-06T20:15:39,041 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 5579bc46ccc141da803e6e4322392a87, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1694031336377 2023-09-06T20:15:39,041 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 6b75d3478c8c4cbd9c0633103b24af6a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1694031337520 2023-09-06T20:15:39,047 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#C#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:39,048 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5d57636bfe1d4f159972e307b570ed3e is 50, key is test_row_0/C:col10/1694031337521/Put/seqid=0 2023-09-06T20:15:39,066 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5d57636bfe1d4f159972e307b570ed3e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5d57636bfe1d4f159972e307b570ed3e 2023-09-06T20:15:39,070 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/C of 969b36246b1411b7cf0e9c1debb5278b into 5d57636bfe1d4f159972e307b570ed3e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:39,070 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:39,070 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/C, priority=12, startTime=1694031338996; duration=0sec 2023-09-06T20:15:39,070 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:39,070 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:C 2023-09-06T20:15:39,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2023-09-06T20:15:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:39,099 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2023-09-06T20:15:39,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:39,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:39,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:39,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:39,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:39,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:39,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/cc887c10f85b40be89b0571111fb8903 is 50, key is test_row_0/A:col10/1694031338166/Put/seqid=0 2023-09-06T20:15:39,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:39,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031399304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031399305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031399306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031399307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031399308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031399410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031399410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031399411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031399413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031399415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,432 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/34edbd77ac6142dd920468d9e94af21a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/34edbd77ac6142dd920468d9e94af21a 2023-09-06T20:15:39,437 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/A of 969b36246b1411b7cf0e9c1debb5278b into 34edbd77ac6142dd920468d9e94af21a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:39,437 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:39,437 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/A, priority=12, startTime=1694031338996; duration=0sec 2023-09-06T20:15:39,437 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:39,437 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:A 2023-09-06T20:15:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-06T20:15:39,507 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/cc887c10f85b40be89b0571111fb8903 2023-09-06T20:15:39,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/1706ca118121493d8d3b64da1602925f is 50, key is test_row_0/B:col10/1694031338166/Put/seqid=0 2023-09-06T20:15:39,520 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/1706ca118121493d8d3b64da1602925f 2023-09-06T20:15:39,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/67b1e7fc7b334f8db5a99d50583ce0a0 is 50, key is test_row_0/C:col10/1694031338166/Put/seqid=0 2023-09-06T20:15:39,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031399615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031399615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031399616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031399619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031399620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031399918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031399920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031399922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031399925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031399926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:39,940 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/67b1e7fc7b334f8db5a99d50583ce0a0 2023-09-06T20:15:39,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/cc887c10f85b40be89b0571111fb8903 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/cc887c10f85b40be89b0571111fb8903 2023-09-06T20:15:39,949 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/cc887c10f85b40be89b0571111fb8903, entries=150, sequenceid=192, filesize=11.8 K 2023-09-06T20:15:39,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/1706ca118121493d8d3b64da1602925f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/1706ca118121493d8d3b64da1602925f 2023-09-06T20:15:39,953 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/1706ca118121493d8d3b64da1602925f, entries=150, sequenceid=192, filesize=11.8 K 2023-09-06T20:15:39,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/67b1e7fc7b334f8db5a99d50583ce0a0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/67b1e7fc7b334f8db5a99d50583ce0a0 2023-09-06T20:15:39,959 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/67b1e7fc7b334f8db5a99d50583ce0a0, entries=150, sequenceid=192, filesize=11.8 K 2023-09-06T20:15:39,960 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3022): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 969b36246b1411b7cf0e9c1debb5278b in 861ms, sequenceid=192, compaction requested=false 2023-09-06T20:15:39,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:39,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:39,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=143 2023-09-06T20:15:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=143 2023-09-06T20:15:39,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=143, resume processing ppid=142 2023-09-06T20:15:39,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=143, ppid=142, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6230 sec 2023-09-06T20:15:39,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=142, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.6270 sec 2023-09-06T20:15:40,185 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/ffacdf7c464346b38a66eddb6017ee74, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e9dc51eaa3794c6da7e736ce9e20234f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/afa13eca4005415bb205ec70c02929a6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/462c4428bbb74f0db0ad093edc0cbe52, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/5902983c674b4116b37d9becf7722b21, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/9e75067f938a4078a34a84e960281086, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/231cfa8a337d4758b0ecd27bbbd9bf4b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/3d22221596a24702a46675cc1b05f7cd, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bfe9e92664f64bbe9739e7d274492958, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/d41fec7eeeca4bd0af2b7eee0e9a1863, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/729bc5b390664aa2818a60358810253a] to archive 2023-09-06T20:15:40,186 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:15:40,188 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/ffacdf7c464346b38a66eddb6017ee74 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/ffacdf7c464346b38a66eddb6017ee74 2023-09-06T20:15:40,190 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e9dc51eaa3794c6da7e736ce9e20234f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e9dc51eaa3794c6da7e736ce9e20234f 2023-09-06T20:15:40,191 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/afa13eca4005415bb205ec70c02929a6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/afa13eca4005415bb205ec70c02929a6 2023-09-06T20:15:40,192 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/462c4428bbb74f0db0ad093edc0cbe52 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/462c4428bbb74f0db0ad093edc0cbe52 2023-09-06T20:15:40,193 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/5902983c674b4116b37d9becf7722b21 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/5902983c674b4116b37d9becf7722b21 2023-09-06T20:15:40,195 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/9e75067f938a4078a34a84e960281086 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/9e75067f938a4078a34a84e960281086 2023-09-06T20:15:40,196 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/231cfa8a337d4758b0ecd27bbbd9bf4b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/231cfa8a337d4758b0ecd27bbbd9bf4b 2023-09-06T20:15:40,198 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/3d22221596a24702a46675cc1b05f7cd to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/3d22221596a24702a46675cc1b05f7cd 2023-09-06T20:15:40,199 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bfe9e92664f64bbe9739e7d274492958 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bfe9e92664f64bbe9739e7d274492958 2023-09-06T20:15:40,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/d41fec7eeeca4bd0af2b7eee0e9a1863 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/d41fec7eeeca4bd0af2b7eee0e9a1863 2023-09-06T20:15:40,202 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/729bc5b390664aa2818a60358810253a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/729bc5b390664aa2818a60358810253a 2023-09-06T20:15:40,227 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/36329a47593e462fbf092f3bef1b481c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8f3775be6c604eada3a253f8d04c645b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/6f50d857313940d1ad599e1b6e1d7def, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/83a068dbb52149b884e1d87c8972bf14, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/658c09e67d0a47b8bcadca37245dd9de, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/03cb71a1225a444b9fc4e4260fda29c3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/97131f0636af4001973473b63d869c1c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/966836b14601421ab8ea6554d1c5057e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8010908f08594ab5966afe66188477c4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/7a133b1a513e481097ba13bef642a74d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/2fa43439abd6495cbea030f351904069] to archive 2023-09-06T20:15:40,228 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:15:40,233 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/36329a47593e462fbf092f3bef1b481c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/36329a47593e462fbf092f3bef1b481c 2023-09-06T20:15:40,235 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8f3775be6c604eada3a253f8d04c645b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8f3775be6c604eada3a253f8d04c645b 2023-09-06T20:15:40,237 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/6f50d857313940d1ad599e1b6e1d7def to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/6f50d857313940d1ad599e1b6e1d7def 2023-09-06T20:15:40,239 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/83a068dbb52149b884e1d87c8972bf14 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/83a068dbb52149b884e1d87c8972bf14 2023-09-06T20:15:40,240 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/658c09e67d0a47b8bcadca37245dd9de to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/658c09e67d0a47b8bcadca37245dd9de 2023-09-06T20:15:40,242 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/03cb71a1225a444b9fc4e4260fda29c3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/03cb71a1225a444b9fc4e4260fda29c3 2023-09-06T20:15:40,244 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/97131f0636af4001973473b63d869c1c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/97131f0636af4001973473b63d869c1c 2023-09-06T20:15:40,246 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/966836b14601421ab8ea6554d1c5057e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/966836b14601421ab8ea6554d1c5057e 2023-09-06T20:15:40,248 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8010908f08594ab5966afe66188477c4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8010908f08594ab5966afe66188477c4 2023-09-06T20:15:40,250 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/7a133b1a513e481097ba13bef642a74d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/7a133b1a513e481097ba13bef642a74d 2023-09-06T20:15:40,252 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/2fa43439abd6495cbea030f351904069 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/2fa43439abd6495cbea030f351904069 2023-09-06T20:15:40,256 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/834244a735224e6fa2bacbd1291f29b9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a16fdff61596487d97192e05c60887a4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a32dc0695c0045e882fcb317f0f1b3b6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/fe9a997c83be4d7b8c70dddced91f1d0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/9986b3b934af4c67ae393183e0bc8d86, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/349326de8d024c9684891754857073e3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/472abb726fa643c3a295a43052fd3804, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/17cd08f7fb164e21b1028544f7d1922c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/9bab74e5f2654f9bacba8e1db1528608, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5579bc46ccc141da803e6e4322392a87, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/6b75d3478c8c4cbd9c0633103b24af6a] to archive 2023-09-06T20:15:40,257 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:15:40,259 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/834244a735224e6fa2bacbd1291f29b9 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/834244a735224e6fa2bacbd1291f29b9 2023-09-06T20:15:40,261 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a16fdff61596487d97192e05c60887a4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a16fdff61596487d97192e05c60887a4 2023-09-06T20:15:40,262 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a32dc0695c0045e882fcb317f0f1b3b6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a32dc0695c0045e882fcb317f0f1b3b6 2023-09-06T20:15:40,263 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/fe9a997c83be4d7b8c70dddced91f1d0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/fe9a997c83be4d7b8c70dddced91f1d0 2023-09-06T20:15:40,265 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/9986b3b934af4c67ae393183e0bc8d86 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/9986b3b934af4c67ae393183e0bc8d86 2023-09-06T20:15:40,266 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/349326de8d024c9684891754857073e3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/349326de8d024c9684891754857073e3 2023-09-06T20:15:40,267 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/472abb726fa643c3a295a43052fd3804 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/472abb726fa643c3a295a43052fd3804 2023-09-06T20:15:40,268 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/17cd08f7fb164e21b1028544f7d1922c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/17cd08f7fb164e21b1028544f7d1922c 2023-09-06T20:15:40,269 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/9bab74e5f2654f9bacba8e1db1528608 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/9bab74e5f2654f9bacba8e1db1528608 2023-09-06T20:15:40,270 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5579bc46ccc141da803e6e4322392a87 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5579bc46ccc141da803e6e4322392a87 2023-09-06T20:15:40,271 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/6b75d3478c8c4cbd9c0633103b24af6a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/6b75d3478c8c4cbd9c0633103b24af6a 2023-09-06T20:15:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:40,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2023-09-06T20:15:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:40,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/b462f22c06df4c19bdcda6c63addb031 is 50, key is test_row_0/A:col10/1694031340426/Put/seqid=0 2023-09-06T20:15:40,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/b462f22c06df4c19bdcda6c63addb031 2023-09-06T20:15:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-06T20:15:40,441 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 142 completed 2023-09-06T20:15:40,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2023-09-06T20:15:40,445 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:40,446 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:40,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=144 2023-09-06T20:15:40,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/5d4f49e3e389417dbf531ab0e71db939 is 50, key is test_row_0/B:col10/1694031340426/Put/seqid=0 2023-09-06T20:15:40,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031400457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031400464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031400464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031400465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031400466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/5d4f49e3e389417dbf531ab0e71db939 2023-09-06T20:15:40,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/a12abad821ad4e4cbc7173181963d6ce is 50, key is test_row_0/C:col10/1694031340426/Put/seqid=0 2023-09-06T20:15:40,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/a12abad821ad4e4cbc7173181963d6ce 2023-09-06T20:15:40,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/b462f22c06df4c19bdcda6c63addb031 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/b462f22c06df4c19bdcda6c63addb031 2023-09-06T20:15:40,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/b462f22c06df4c19bdcda6c63addb031, entries=200, sequenceid=210, filesize=14.2 K 2023-09-06T20:15:40,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/5d4f49e3e389417dbf531ab0e71db939 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/5d4f49e3e389417dbf531ab0e71db939 2023-09-06T20:15:40,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/5d4f49e3e389417dbf531ab0e71db939, entries=150, sequenceid=210, filesize=11.8 K 2023-09-06T20:15:40,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/a12abad821ad4e4cbc7173181963d6ce as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a12abad821ad4e4cbc7173181963d6ce 2023-09-06T20:15:40,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a12abad821ad4e4cbc7173181963d6ce, entries=150, sequenceid=210, filesize=11.8 K 2023-09-06T20:15:40,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 969b36246b1411b7cf0e9c1debb5278b in 70ms, sequenceid=210, compaction requested=true 2023-09-06T20:15:40,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:40,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:40,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:40,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:40,497 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:40,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:40,497 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:40,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:40,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:40,498 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39168 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:40,498 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/A is initiating minor compaction (all files) 2023-09-06T20:15:40,499 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36778 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:40,499 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/A in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:40,499 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/B is initiating minor compaction (all files) 2023-09-06T20:15:40,499 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/B in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:40,499 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/34edbd77ac6142dd920468d9e94af21a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/cc887c10f85b40be89b0571111fb8903, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/b462f22c06df4c19bdcda6c63addb031] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=38.3 K 2023-09-06T20:15:40,499 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/eda06905d26f4fdda6348f321e0c08c7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/1706ca118121493d8d3b64da1602925f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/5d4f49e3e389417dbf531ab0e71db939] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=35.9 K 2023-09-06T20:15:40,499 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 34edbd77ac6142dd920468d9e94af21a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1694031337520 2023-09-06T20:15:40,499 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting eda06905d26f4fdda6348f321e0c08c7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1694031337520 2023-09-06T20:15:40,499 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting cc887c10f85b40be89b0571111fb8903, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1694031338164 2023-09-06T20:15:40,499 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 1706ca118121493d8d3b64da1602925f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1694031338164 2023-09-06T20:15:40,500 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting b462f22c06df4c19bdcda6c63addb031, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1694031339303 2023-09-06T20:15:40,500 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 5d4f49e3e389417dbf531ab0e71db939, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1694031339303 2023-09-06T20:15:40,507 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#A#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:40,507 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/f235a65e37c84a11908eba892cbf1335 is 50, key is test_row_0/A:col10/1694031340426/Put/seqid=0 2023-09-06T20:15:40,508 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#B#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:40,509 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/3826c9b1f9124ec6b5f4b51f17b6cffa is 50, key is test_row_0/B:col10/1694031340426/Put/seqid=0 2023-09-06T20:15:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=144 2023-09-06T20:15:40,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:40,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2023-09-06T20:15:40,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:40,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:40,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:40,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:40,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:40,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:40,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/12501d23d4564b169c1bd28b7540e2b4 is 50, key is test_row_0/A:col10/1694031340454/Put/seqid=0 2023-09-06T20:15:40,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031400588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031400588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031400590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031400591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031400594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2023-09-06T20:15:40,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:40,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:40,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:40,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(53): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:40,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:40,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031400695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031400696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031400696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031400696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031400696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=144 2023-09-06T20:15:40,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2023-09-06T20:15:40,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:40,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:40,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:40,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(53): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:40,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:40,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2023-09-06T20:15:40,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:40,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:40,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:40,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(53): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:40,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:40,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031400902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031400903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031400903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031400903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031400903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:40,928 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/f235a65e37c84a11908eba892cbf1335 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/f235a65e37c84a11908eba892cbf1335 2023-09-06T20:15:40,931 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/3826c9b1f9124ec6b5f4b51f17b6cffa as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/3826c9b1f9124ec6b5f4b51f17b6cffa 2023-09-06T20:15:40,933 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/A of 969b36246b1411b7cf0e9c1debb5278b into f235a65e37c84a11908eba892cbf1335(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:40,933 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:40,933 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/A, priority=13, startTime=1694031340497; duration=0sec 2023-09-06T20:15:40,934 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:40,934 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:A 2023-09-06T20:15:40,934 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:40,935 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36778 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:40,935 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/C is initiating minor compaction (all files) 2023-09-06T20:15:40,935 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/C in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:40,935 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5d57636bfe1d4f159972e307b570ed3e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/67b1e7fc7b334f8db5a99d50583ce0a0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a12abad821ad4e4cbc7173181963d6ce] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=35.9 K 2023-09-06T20:15:40,935 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5d57636bfe1d4f159972e307b570ed3e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1694031337520 2023-09-06T20:15:40,936 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 67b1e7fc7b334f8db5a99d50583ce0a0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1694031338164 2023-09-06T20:15:40,936 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting a12abad821ad4e4cbc7173181963d6ce, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1694031339303 2023-09-06T20:15:40,936 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/B of 969b36246b1411b7cf0e9c1debb5278b into 3826c9b1f9124ec6b5f4b51f17b6cffa(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:40,936 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:40,936 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/B, priority=13, startTime=1694031340497; duration=0sec 2023-09-06T20:15:40,936 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:40,936 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:B 2023-09-06T20:15:40,941 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#C#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:40,941 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/b14a23c51e2e4001bfde2f5e416fbd3e is 50, key is test_row_0/C:col10/1694031340426/Put/seqid=0 2023-09-06T20:15:40,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/12501d23d4564b169c1bd28b7540e2b4 2023-09-06T20:15:40,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/8450b64ad80643b78c307f8ddfb5b458 is 50, key is test_row_0/B:col10/1694031340454/Put/seqid=0 2023-09-06T20:15:40,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/8450b64ad80643b78c307f8ddfb5b458 2023-09-06T20:15:40,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/188b53f56434403b9c7c7a0406b203e8 is 50, key is test_row_0/C:col10/1694031340454/Put/seqid=0 2023-09-06T20:15:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=144 2023-09-06T20:15:41,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2023-09-06T20:15:41,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:41,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:41,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:41,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(53): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:41,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:41,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2023-09-06T20:15:41,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:41,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:41,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:41,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(53): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:41,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:41,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031401207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031401208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031401208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031401209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031401209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,350 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/b14a23c51e2e4001bfde2f5e416fbd3e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/b14a23c51e2e4001bfde2f5e416fbd3e 2023-09-06T20:15:41,353 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/C of 969b36246b1411b7cf0e9c1debb5278b into b14a23c51e2e4001bfde2f5e416fbd3e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:41,354 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:41,354 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/C, priority=13, startTime=1694031340497; duration=0sec 2023-09-06T20:15:41,354 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:41,354 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:C 2023-09-06T20:15:41,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2023-09-06T20:15:41,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:41,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:41,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:41,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(53): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:41,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:41,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/188b53f56434403b9c7c7a0406b203e8 2023-09-06T20:15:41,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/12501d23d4564b169c1bd28b7540e2b4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/12501d23d4564b169c1bd28b7540e2b4 2023-09-06T20:15:41,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/12501d23d4564b169c1bd28b7540e2b4, entries=200, sequenceid=230, filesize=14.2 K 2023-09-06T20:15:41,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/8450b64ad80643b78c307f8ddfb5b458 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8450b64ad80643b78c307f8ddfb5b458 2023-09-06T20:15:41,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8450b64ad80643b78c307f8ddfb5b458, entries=150, sequenceid=230, filesize=11.8 K 2023-09-06T20:15:41,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/188b53f56434403b9c7c7a0406b203e8 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/188b53f56434403b9c7c7a0406b203e8 2023-09-06T20:15:41,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/188b53f56434403b9c7c7a0406b203e8, entries=150, sequenceid=230, filesize=11.8 K 2023-09-06T20:15:41,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 969b36246b1411b7cf0e9c1debb5278b in 851ms, sequenceid=230, compaction requested=false 2023-09-06T20:15:41,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:41,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2023-09-06T20:15:41,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:41,514 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2023-09-06T20:15:41,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:41,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/4ce70c85c46d4646b740c1c039248fc7 is 50, key is test_row_0/A:col10/1694031340587/Put/seqid=0 2023-09-06T20:15:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=144 2023-09-06T20:15:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:41,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:41,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031401733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031401734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031401735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031401737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031401737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031401840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031401845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031401845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031401845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031401845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:41,927 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/4ce70c85c46d4646b740c1c039248fc7 2023-09-06T20:15:41,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/ae08c4d4b0014692b146beda0501d5b9 is 50, key is test_row_0/B:col10/1694031340587/Put/seqid=0 2023-09-06T20:15:42,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031402046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031402053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031402053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031402053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031402053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,344 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/ae08c4d4b0014692b146beda0501d5b9 2023-09-06T20:15:42,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/ca9024745bd5403b8710cd6eaac37a9a is 50, key is test_row_0/C:col10/1694031340587/Put/seqid=0 2023-09-06T20:15:42,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031402353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,360 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/ca9024745bd5403b8710cd6eaac37a9a 2023-09-06T20:15:42,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031402357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031402358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/4ce70c85c46d4646b740c1c039248fc7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/4ce70c85c46d4646b740c1c039248fc7 2023-09-06T20:15:42,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031402359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031402359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,366 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/4ce70c85c46d4646b740c1c039248fc7, entries=150, sequenceid=249, filesize=11.8 K 2023-09-06T20:15:42,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/ae08c4d4b0014692b146beda0501d5b9 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/ae08c4d4b0014692b146beda0501d5b9 2023-09-06T20:15:42,370 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/ae08c4d4b0014692b146beda0501d5b9, entries=150, sequenceid=249, filesize=11.8 K 2023-09-06T20:15:42,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/ca9024745bd5403b8710cd6eaac37a9a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/ca9024745bd5403b8710cd6eaac37a9a 2023-09-06T20:15:42,376 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/ca9024745bd5403b8710cd6eaac37a9a, entries=150, sequenceid=249, filesize=11.8 K 2023-09-06T20:15:42,377 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 969b36246b1411b7cf0e9c1debb5278b in 863ms, sequenceid=249, compaction requested=true 2023-09-06T20:15:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=145 2023-09-06T20:15:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=145 2023-09-06T20:15:42,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=145, resume processing ppid=144 2023-09-06T20:15:42,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=145, ppid=144, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9320 sec 2023-09-06T20:15:42,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=144, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.9360 sec 2023-09-06T20:15:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=144 2023-09-06T20:15:42,553 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 144 completed 2023-09-06T20:15:42,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2023-09-06T20:15:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=146 2023-09-06T20:15:42,557 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:42,558 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:42,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=146 2023-09-06T20:15:42,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2023-09-06T20:15:42,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:42,712 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2023-09-06T20:15:42,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:42,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:42,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:42,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:42,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:42,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:42,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/1c0fcb18f2564556b6ded88ab21cc2e8 is 50, key is test_row_0/A:col10/1694031341735/Put/seqid=0 2023-09-06T20:15:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=146 2023-09-06T20:15:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:42,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:42,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031402876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031402877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031402878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031402881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031402882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031402987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031402987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031402988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031402988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:42,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031402989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:43,119 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/1c0fcb18f2564556b6ded88ab21cc2e8 2023-09-06T20:15:43,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/503f8413d2234464afe18b56f1348a10 is 50, key is test_row_0/B:col10/1694031341735/Put/seqid=0 2023-09-06T20:15:43,131 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/503f8413d2234464afe18b56f1348a10 2023-09-06T20:15:43,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/bdd4f68767094283b1e47be11f4576b9 is 50, key is test_row_0/C:col10/1694031341735/Put/seqid=0 2023-09-06T20:15:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=146 2023-09-06T20:15:43,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031403192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:43,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031403193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:43,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031403193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:43,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031403194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:43,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031403196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:43,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031403495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:43,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031403499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:43,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031403500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:43,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031403500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:43,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031403502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:43,540 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/bdd4f68767094283b1e47be11f4576b9 2023-09-06T20:15:43,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/1c0fcb18f2564556b6ded88ab21cc2e8 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/1c0fcb18f2564556b6ded88ab21cc2e8 2023-09-06T20:15:43,548 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/1c0fcb18f2564556b6ded88ab21cc2e8, entries=150, sequenceid=270, filesize=12.0 K 2023-09-06T20:15:43,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/503f8413d2234464afe18b56f1348a10 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/503f8413d2234464afe18b56f1348a10 2023-09-06T20:15:43,554 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/503f8413d2234464afe18b56f1348a10, entries=150, sequenceid=270, filesize=12.0 K 2023-09-06T20:15:43,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/bdd4f68767094283b1e47be11f4576b9 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/bdd4f68767094283b1e47be11f4576b9 2023-09-06T20:15:43,558 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/bdd4f68767094283b1e47be11f4576b9, entries=150, sequenceid=270, filesize=12.0 K 2023-09-06T20:15:43,558 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3022): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 969b36246b1411b7cf0e9c1debb5278b in 846ms, sequenceid=270, compaction requested=true 2023-09-06T20:15:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:43,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:43,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=147 2023-09-06T20:15:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=147 2023-09-06T20:15:43,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=147, resume processing ppid=146 2023-09-06T20:15:43,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=147, ppid=146, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0010 sec 2023-09-06T20:15:43,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=146, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.0120 sec 2023-09-06T20:15:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=146 2023-09-06T20:15:43,660 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 146 completed 2023-09-06T20:15:43,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2023-09-06T20:15:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=148 2023-09-06T20:15:43,664 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:43,664 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:43,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=148 2023-09-06T20:15:43,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2023-09-06T20:15:43,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:43,817 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-06T20:15:43,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:43,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:43,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:43,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:43,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:43,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:43,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/e23611564d964a71aee8d00037e0c4c6 is 50, key is test_row_0/A:col10/1694031342876/Put/seqid=0 2023-09-06T20:15:43,827 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/e23611564d964a71aee8d00037e0c4c6 2023-09-06T20:15:43,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/c4aa6a1bd3d24d8b87ca3732f5ebff05 is 50, key is test_row_0/B:col10/1694031342876/Put/seqid=0 2023-09-06T20:15:43,842 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/c4aa6a1bd3d24d8b87ca3732f5ebff05 2023-09-06T20:15:43,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/73451464d4c14568b3a400f77974f09a is 50, key is test_row_0/C:col10/1694031342876/Put/seqid=0 2023-09-06T20:15:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=148 2023-09-06T20:15:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:44,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:44,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031404031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031404032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031404033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031404039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031404041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031404141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031404142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031404142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031404146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031404148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,257 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/73451464d4c14568b3a400f77974f09a 2023-09-06T20:15:44,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/e23611564d964a71aee8d00037e0c4c6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e23611564d964a71aee8d00037e0c4c6 2023-09-06T20:15:44,266 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e23611564d964a71aee8d00037e0c4c6, entries=150, sequenceid=285, filesize=12.0 K 2023-09-06T20:15:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=148 2023-09-06T20:15:44,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/c4aa6a1bd3d24d8b87ca3732f5ebff05 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/c4aa6a1bd3d24d8b87ca3732f5ebff05 2023-09-06T20:15:44,272 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/c4aa6a1bd3d24d8b87ca3732f5ebff05, entries=150, sequenceid=285, filesize=12.0 K 2023-09-06T20:15:44,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/73451464d4c14568b3a400f77974f09a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/73451464d4c14568b3a400f77974f09a 2023-09-06T20:15:44,277 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/73451464d4c14568b3a400f77974f09a, entries=150, sequenceid=285, filesize=12.0 K 2023-09-06T20:15:44,278 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 969b36246b1411b7cf0e9c1debb5278b in 461ms, sequenceid=285, compaction requested=true 2023-09-06T20:15:44,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:44,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:44,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=149 2023-09-06T20:15:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=149 2023-09-06T20:15:44,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=149, resume processing ppid=148 2023-09-06T20:15:44,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=149, ppid=148, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 615 msec 2023-09-06T20:15:44,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=148, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 622 msec 2023-09-06T20:15:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:44,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2023-09-06T20:15:44,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:44,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:44,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:44,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:44,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:44,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:44,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/e5d5267801984573bad28502bb6d635e is 50, key is test_row_0/A:col10/1694031344038/Put/seqid=0 2023-09-06T20:15:44,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/e5d5267801984573bad28502bb6d635e 2023-09-06T20:15:44,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/455a82c1b9794959a18f92e71b5b2134 is 50, key is test_row_0/B:col10/1694031344038/Put/seqid=0 2023-09-06T20:15:44,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031404368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031404369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031404375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031404376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031404377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031404477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031404477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031404482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031404482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031404483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031404684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031404684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031404689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031404690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031404691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=148 2023-09-06T20:15:44,767 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 148 completed 2023-09-06T20:15:44,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2023-09-06T20:15:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=150 2023-09-06T20:15:44,771 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:44,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/455a82c1b9794959a18f92e71b5b2134 2023-09-06T20:15:44,772 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:44,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:44,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/48741bacb05b488c8788c928c6e40ebb is 50, key is test_row_0/C:col10/1694031344038/Put/seqid=0 2023-09-06T20:15:44,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/48741bacb05b488c8788c928c6e40ebb 2023-09-06T20:15:44,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/e5d5267801984573bad28502bb6d635e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e5d5267801984573bad28502bb6d635e 2023-09-06T20:15:44,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e5d5267801984573bad28502bb6d635e, entries=200, sequenceid=307, filesize=14.4 K 2023-09-06T20:15:44,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/455a82c1b9794959a18f92e71b5b2134 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/455a82c1b9794959a18f92e71b5b2134 2023-09-06T20:15:44,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/455a82c1b9794959a18f92e71b5b2134, entries=150, sequenceid=307, filesize=12.0 K 2023-09-06T20:15:44,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/48741bacb05b488c8788c928c6e40ebb as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/48741bacb05b488c8788c928c6e40ebb 2023-09-06T20:15:44,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/48741bacb05b488c8788c928c6e40ebb, entries=150, sequenceid=307, filesize=12.0 K 2023-09-06T20:15:44,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 969b36246b1411b7cf0e9c1debb5278b in 452ms, sequenceid=307, compaction requested=true 2023-09-06T20:15:44,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:44,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:44,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:44,803 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2023-09-06T20:15:44,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:44,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:44,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:44,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-06T20:15:44,803 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2023-09-06T20:15:44,805 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 78187 starting at candidate #0 after considering 10 permutations with 10 in ratio 2023-09-06T20:15:44,805 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/A is initiating minor compaction (all files) 2023-09-06T20:15:44,805 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/A in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:44,805 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73357 starting at candidate #0 after considering 10 permutations with 10 in ratio 2023-09-06T20:15:44,805 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/B is initiating minor compaction (all files) 2023-09-06T20:15:44,805 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/f235a65e37c84a11908eba892cbf1335, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/12501d23d4564b169c1bd28b7540e2b4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/4ce70c85c46d4646b740c1c039248fc7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/1c0fcb18f2564556b6ded88ab21cc2e8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e23611564d964a71aee8d00037e0c4c6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e5d5267801984573bad28502bb6d635e] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=76.4 K 2023-09-06T20:15:44,805 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/B in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:44,805 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/3826c9b1f9124ec6b5f4b51f17b6cffa, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8450b64ad80643b78c307f8ddfb5b458, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/ae08c4d4b0014692b146beda0501d5b9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/503f8413d2234464afe18b56f1348a10, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/c4aa6a1bd3d24d8b87ca3732f5ebff05, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/455a82c1b9794959a18f92e71b5b2134] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=71.6 K 2023-09-06T20:15:44,806 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting f235a65e37c84a11908eba892cbf1335, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1694031339303 2023-09-06T20:15:44,810 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 3826c9b1f9124ec6b5f4b51f17b6cffa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1694031339303 2023-09-06T20:15:44,810 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 12501d23d4564b169c1bd28b7540e2b4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1694031340454 2023-09-06T20:15:44,810 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 8450b64ad80643b78c307f8ddfb5b458, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1694031340454 2023-09-06T20:15:44,810 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4ce70c85c46d4646b740c1c039248fc7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1694031340587 2023-09-06T20:15:44,811 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting ae08c4d4b0014692b146beda0501d5b9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1694031340587 2023-09-06T20:15:44,811 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1c0fcb18f2564556b6ded88ab21cc2e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1694031341731 2023-09-06T20:15:44,811 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 503f8413d2234464afe18b56f1348a10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1694031341731 2023-09-06T20:15:44,811 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting e23611564d964a71aee8d00037e0c4c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1694031342876 2023-09-06T20:15:44,811 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting c4aa6a1bd3d24d8b87ca3732f5ebff05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1694031342876 2023-09-06T20:15:44,812 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting e5d5267801984573bad28502bb6d635e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1694031344032 2023-09-06T20:15:44,812 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 455a82c1b9794959a18f92e71b5b2134, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1694031344032 2023-09-06T20:15:44,824 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#A#compaction#481 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:44,825 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/0335297fb4a741e7aec925eee93a707d is 50, key is test_row_0/A:col10/1694031344038/Put/seqid=0 2023-09-06T20:15:44,828 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#B#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:44,828 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/f411f22ec5f245889508c5465764e993 is 50, key is test_row_0/B:col10/1694031344038/Put/seqid=0 2023-09-06T20:15:44,843 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/0335297fb4a741e7aec925eee93a707d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/0335297fb4a741e7aec925eee93a707d 2023-09-06T20:15:44,848 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 6 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/A of 969b36246b1411b7cf0e9c1debb5278b into 0335297fb4a741e7aec925eee93a707d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:44,848 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:44,848 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/A, priority=10, startTime=1694031344803; duration=0sec 2023-09-06T20:15:44,848 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:44,848 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:A 2023-09-06T20:15:44,848 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2023-09-06T20:15:44,851 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/f411f22ec5f245889508c5465764e993 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/f411f22ec5f245889508c5465764e993 2023-09-06T20:15:44,852 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73357 starting at candidate #0 after considering 10 permutations with 10 in ratio 2023-09-06T20:15:44,852 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/C is initiating minor compaction (all files) 2023-09-06T20:15:44,852 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/C in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:44,852 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/b14a23c51e2e4001bfde2f5e416fbd3e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/188b53f56434403b9c7c7a0406b203e8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/ca9024745bd5403b8710cd6eaac37a9a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/bdd4f68767094283b1e47be11f4576b9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/73451464d4c14568b3a400f77974f09a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/48741bacb05b488c8788c928c6e40ebb] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=71.6 K 2023-09-06T20:15:44,852 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting b14a23c51e2e4001bfde2f5e416fbd3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1694031339303 2023-09-06T20:15:44,857 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 188b53f56434403b9c7c7a0406b203e8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1694031340454 2023-09-06T20:15:44,858 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting ca9024745bd5403b8710cd6eaac37a9a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1694031340587 2023-09-06T20:15:44,858 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 6 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/B of 969b36246b1411b7cf0e9c1debb5278b into f411f22ec5f245889508c5465764e993(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:44,858 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:44,858 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/B, priority=10, startTime=1694031344803; duration=0sec 2023-09-06T20:15:44,858 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:44,858 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:B 2023-09-06T20:15:44,858 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting bdd4f68767094283b1e47be11f4576b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1694031341731 2023-09-06T20:15:44,858 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 73451464d4c14568b3a400f77974f09a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1694031342876 2023-09-06T20:15:44,858 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 48741bacb05b488c8788c928c6e40ebb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1694031344032 2023-09-06T20:15:44,867 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#C#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:44,868 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5dfa56afa1b94a63a781e5f87498788c is 50, key is test_row_0/C:col10/1694031344038/Put/seqid=0 2023-09-06T20:15:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=150 2023-09-06T20:15:44,891 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5dfa56afa1b94a63a781e5f87498788c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5dfa56afa1b94a63a781e5f87498788c 2023-09-06T20:15:44,896 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 6 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/C of 969b36246b1411b7cf0e9c1debb5278b into 5dfa56afa1b94a63a781e5f87498788c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:44,896 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:44,896 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/C, priority=10, startTime=1694031344803; duration=0sec 2023-09-06T20:15:44,896 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:44,896 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:C 2023-09-06T20:15:44,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2023-09-06T20:15:44,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:44,933 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-06T20:15:44,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:44,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:44,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:44,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:44,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:44,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:44,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/5dec7d14f9b84e2faf6030a60b34cc7c is 50, key is test_row_0/A:col10/1694031344375/Put/seqid=0 2023-09-06T20:15:44,951 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/5dec7d14f9b84e2faf6030a60b34cc7c 2023-09-06T20:15:44,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/98475f353a0741e0bed1e62c7c0aa25a is 50, key is test_row_0/B:col10/1694031344375/Put/seqid=0 2023-09-06T20:15:44,967 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/98475f353a0741e0bed1e62c7c0aa25a 2023-09-06T20:15:44,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5f38c0005a744187a593f2a6156d25de is 50, key is test_row_0/C:col10/1694031344375/Put/seqid=0 2023-09-06T20:15:44,980 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5f38c0005a744187a593f2a6156d25de 2023-09-06T20:15:44,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/5dec7d14f9b84e2faf6030a60b34cc7c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/5dec7d14f9b84e2faf6030a60b34cc7c 2023-09-06T20:15:44,989 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/5dec7d14f9b84e2faf6030a60b34cc7c, entries=150, sequenceid=325, filesize=12.0 K 2023-09-06T20:15:44,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/98475f353a0741e0bed1e62c7c0aa25a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/98475f353a0741e0bed1e62c7c0aa25a 2023-09-06T20:15:44,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:44,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:44,993 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/98475f353a0741e0bed1e62c7c0aa25a, entries=150, sequenceid=325, filesize=12.0 K 2023-09-06T20:15:44,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5f38c0005a744187a593f2a6156d25de as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5f38c0005a744187a593f2a6156d25de 2023-09-06T20:15:44,998 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5f38c0005a744187a593f2a6156d25de, entries=150, sequenceid=325, filesize=12.0 K 2023-09-06T20:15:44,999 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=26.84 KB/27480 for 969b36246b1411b7cf0e9c1debb5278b in 66ms, sequenceid=325, compaction requested=false 2023-09-06T20:15:44,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:44,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:44,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=151 2023-09-06T20:15:45,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=151 2023-09-06T20:15:45,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=151, resume processing ppid=150 2023-09-06T20:15:45,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=151, ppid=150, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 228 msec 2023-09-06T20:15:45,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:45,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:15:45,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=150, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 233 msec 2023-09-06T20:15:45,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:45,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:45,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:45,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:45,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:45,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:45,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/bedb9b281f924e259941bb4c4c32cee2 is 50, key is test_row_0/A:col10/1694031345001/Put/seqid=0 2023-09-06T20:15:45,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/bedb9b281f924e259941bb4c4c32cee2 2023-09-06T20:15:45,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/4354bd496d2a4e59b1d55624f32ee55d is 50, key is test_row_0/B:col10/1694031345001/Put/seqid=0 2023-09-06T20:15:45,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/4354bd496d2a4e59b1d55624f32ee55d 2023-09-06T20:15:45,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/538af06c9b064bb4bb9ad4cc245accf3 is 50, key is test_row_0/C:col10/1694031345001/Put/seqid=0 2023-09-06T20:15:45,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/538af06c9b064bb4bb9ad4cc245accf3 2023-09-06T20:15:45,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031405031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031405032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/bedb9b281f924e259941bb4c4c32cee2 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bedb9b281f924e259941bb4c4c32cee2 2023-09-06T20:15:45,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031405032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031405033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bedb9b281f924e259941bb4c4c32cee2, entries=300, sequenceid=337, filesize=19.1 K 2023-09-06T20:15:45,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/4354bd496d2a4e59b1d55624f32ee55d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4354bd496d2a4e59b1d55624f32ee55d 2023-09-06T20:15:45,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031405037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4354bd496d2a4e59b1d55624f32ee55d, entries=150, sequenceid=337, filesize=12.0 K 2023-09-06T20:15:45,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/538af06c9b064bb4bb9ad4cc245accf3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/538af06c9b064bb4bb9ad4cc245accf3 2023-09-06T20:15:45,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/538af06c9b064bb4bb9ad4cc245accf3, entries=150, sequenceid=337, filesize=12.0 K 2023-09-06T20:15:45,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 969b36246b1411b7cf0e9c1debb5278b in 48ms, sequenceid=337, compaction requested=true 2023-09-06T20:15:45,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:45,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:45,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:45,050 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:45,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:45,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:45,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:45,050 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:45,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:45,051 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44480 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:45,051 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37160 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:45,051 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/A is initiating minor compaction (all files) 2023-09-06T20:15:45,051 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/B is initiating minor compaction (all files) 2023-09-06T20:15:45,051 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/B in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:45,051 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/A in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:45,051 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/0335297fb4a741e7aec925eee93a707d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/5dec7d14f9b84e2faf6030a60b34cc7c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bedb9b281f924e259941bb4c4c32cee2] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=43.4 K 2023-09-06T20:15:45,051 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/f411f22ec5f245889508c5465764e993, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/98475f353a0741e0bed1e62c7c0aa25a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4354bd496d2a4e59b1d55624f32ee55d] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=36.3 K 2023-09-06T20:15:45,052 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 0335297fb4a741e7aec925eee93a707d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1694031344032 2023-09-06T20:15:45,052 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting f411f22ec5f245889508c5465764e993, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1694031344032 2023-09-06T20:15:45,053 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 98475f353a0741e0bed1e62c7c0aa25a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1694031344374 2023-09-06T20:15:45,053 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5dec7d14f9b84e2faf6030a60b34cc7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1694031344374 2023-09-06T20:15:45,053 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 4354bd496d2a4e59b1d55624f32ee55d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1694031344989 2023-09-06T20:15:45,054 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting bedb9b281f924e259941bb4c4c32cee2, keycount=300, bloomtype=ROW, size=19.1 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1694031344989 2023-09-06T20:15:45,061 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#B#compaction#490 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:45,061 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/705bf7395879464ea53b0ca05cfcab8b is 50, key is test_row_0/B:col10/1694031345001/Put/seqid=0 2023-09-06T20:15:45,063 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#A#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:45,064 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/e981323ce62b42778f11683ea2258a49 is 50, key is test_row_0/A:col10/1694031345001/Put/seqid=0 2023-09-06T20:15:45,071 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/705bf7395879464ea53b0ca05cfcab8b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/705bf7395879464ea53b0ca05cfcab8b 2023-09-06T20:15:45,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=150 2023-09-06T20:15:45,073 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 150 completed 2023-09-06T20:15:45,075 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2023-09-06T20:15:45,075 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/B of 969b36246b1411b7cf0e9c1debb5278b into 705bf7395879464ea53b0ca05cfcab8b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:45,075 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:45,078 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/B, priority=13, startTime=1694031345050; duration=0sec 2023-09-06T20:15:45,078 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:45,078 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:B 2023-09-06T20:15:45,078 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:15:45,079 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-09-06T20:15:45,079 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:45,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:45,082 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37160 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:15:45,082 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/C is initiating minor compaction (all files) 2023-09-06T20:15:45,082 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/C in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:45,082 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5dfa56afa1b94a63a781e5f87498788c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5f38c0005a744187a593f2a6156d25de, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/538af06c9b064bb4bb9ad4cc245accf3] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=36.3 K 2023-09-06T20:15:45,083 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 5dfa56afa1b94a63a781e5f87498788c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1694031344032 2023-09-06T20:15:45,083 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 5f38c0005a744187a593f2a6156d25de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1694031344374 2023-09-06T20:15:45,084 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 538af06c9b064bb4bb9ad4cc245accf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1694031344989 2023-09-06T20:15:45,089 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#C#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:45,089 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/025da8a1e54c46c8b56070f645e7f65f is 50, key is test_row_0/C:col10/1694031345001/Put/seqid=0 2023-09-06T20:15:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:45,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-06T20:15:45,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:45,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:45,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:45,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:45,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:45,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:45,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/e8be59989dff44828f88f42f757f112b is 50, key is test_row_0/A:col10/1694031345031/Put/seqid=0 2023-09-06T20:15:45,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031405149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031405149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031405151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031405152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031405152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-09-06T20:15:45,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2023-09-06T20:15:45,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:45,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:45,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:45,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(53): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:45,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:45,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031405253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031405253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031405259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031405261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031405261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-09-06T20:15:45,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2023-09-06T20:15:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:45,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(53): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:45,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031405458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031405458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031405464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031405468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,472 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/e981323ce62b42778f11683ea2258a49 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e981323ce62b42778f11683ea2258a49 2023-09-06T20:15:45,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031405469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,477 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/A of 969b36246b1411b7cf0e9c1debb5278b into e981323ce62b42778f11683ea2258a49(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:45,477 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:45,477 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/A, priority=13, startTime=1694031345050; duration=0sec 2023-09-06T20:15:45,477 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:45,477 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:A 2023-09-06T20:15:45,501 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/025da8a1e54c46c8b56070f645e7f65f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/025da8a1e54c46c8b56070f645e7f65f 2023-09-06T20:15:45,505 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/C of 969b36246b1411b7cf0e9c1debb5278b into 025da8a1e54c46c8b56070f645e7f65f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:45,505 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:45,505 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/C, priority=13, startTime=1694031345050; duration=0sec 2023-09-06T20:15:45,505 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:45,505 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:C 2023-09-06T20:15:45,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2023-09-06T20:15:45,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:45,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:45,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:45,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(53): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:45,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:45,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/e8be59989dff44828f88f42f757f112b 2023-09-06T20:15:45,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/10910ca76b0a49c18febbde87504902d is 50, key is test_row_0/B:col10/1694031345031/Put/seqid=0 2023-09-06T20:15:45,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/10910ca76b0a49c18febbde87504902d 2023-09-06T20:15:45,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/4361d0a71493482da9192982363684e7 is 50, key is test_row_0/C:col10/1694031345031/Put/seqid=0 2023-09-06T20:15:45,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/4361d0a71493482da9192982363684e7 2023-09-06T20:15:45,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/e8be59989dff44828f88f42f757f112b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e8be59989dff44828f88f42f757f112b 2023-09-06T20:15:45,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e8be59989dff44828f88f42f757f112b, entries=200, sequenceid=363, filesize=14.4 K 2023-09-06T20:15:45,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/10910ca76b0a49c18febbde87504902d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/10910ca76b0a49c18febbde87504902d 2023-09-06T20:15:45,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/10910ca76b0a49c18febbde87504902d, entries=150, sequenceid=363, filesize=12.0 K 2023-09-06T20:15:45,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/4361d0a71493482da9192982363684e7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/4361d0a71493482da9192982363684e7 2023-09-06T20:15:45,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/4361d0a71493482da9192982363684e7, entries=150, sequenceid=363, filesize=12.0 K 2023-09-06T20:15:45,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 969b36246b1411b7cf0e9c1debb5278b in 519ms, sequenceid=363, compaction requested=false 2023-09-06T20:15:45,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-09-06T20:15:45,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2023-09-06T20:15:45,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:45,693 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:15:45,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:45,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:45,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:45,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:45,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:45,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:45,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/8a86b18679334b8c9782fffb9ecb6511 is 50, key is test_row_0/A:col10/1694031345145/Put/seqid=0 2023-09-06T20:15:45,715 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/8a86b18679334b8c9782fffb9ecb6511 2023-09-06T20:15:45,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/3b311894be194ae3bd6d550d9b67b670 is 50, key is test_row_0/B:col10/1694031345145/Put/seqid=0 2023-09-06T20:15:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:45,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:45,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031405808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031405809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031405810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031405811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031405811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031405913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031405917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031405917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031405917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:45,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031405917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031406117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031406123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031406123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031406123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031406123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,127 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/3b311894be194ae3bd6d550d9b67b670 2023-09-06T20:15:46,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5f4d810d23a942f6bd86ad800cfdcdf4 is 50, key is test_row_0/C:col10/1694031345145/Put/seqid=0 2023-09-06T20:15:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-09-06T20:15:46,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031406425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031406426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031406427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031406427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031406428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,543 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5f4d810d23a942f6bd86ad800cfdcdf4 2023-09-06T20:15:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/8a86b18679334b8c9782fffb9ecb6511 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/8a86b18679334b8c9782fffb9ecb6511 2023-09-06T20:15:46,551 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/8a86b18679334b8c9782fffb9ecb6511, entries=150, sequenceid=376, filesize=12.0 K 2023-09-06T20:15:46,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/3b311894be194ae3bd6d550d9b67b670 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/3b311894be194ae3bd6d550d9b67b670 2023-09-06T20:15:46,563 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/3b311894be194ae3bd6d550d9b67b670, entries=150, sequenceid=376, filesize=12.0 K 2023-09-06T20:15:46,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5f4d810d23a942f6bd86ad800cfdcdf4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5f4d810d23a942f6bd86ad800cfdcdf4 2023-09-06T20:15:46,568 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5f4d810d23a942f6bd86ad800cfdcdf4, entries=150, sequenceid=376, filesize=12.0 K 2023-09-06T20:15:46,569 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 969b36246b1411b7cf0e9c1debb5278b in 876ms, sequenceid=376, compaction requested=true 2023-09-06T20:15:46,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:46,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:46,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=153 2023-09-06T20:15:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=153 2023-09-06T20:15:46,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=153, resume processing ppid=152 2023-09-06T20:15:46,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=153, ppid=152, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4900 sec 2023-09-06T20:15:46,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=152, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.4980 sec 2023-09-06T20:15:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:46,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-06T20:15:46,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:46,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:46,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:46,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:46,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:46,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:46,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/32cf76982e2846b9ae2f0c8e7b9a484d is 50, key is test_row_0/A:col10/1694031345803/Put/seqid=0 2023-09-06T20:15:46,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/32cf76982e2846b9ae2f0c8e7b9a484d 2023-09-06T20:15:46,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031406941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031406942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031406943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031406943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/4abc4761130041b689d0336cc399363e is 50, key is test_row_0/B:col10/1694031345803/Put/seqid=0 2023-09-06T20:15:46,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031406944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:46,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/4abc4761130041b689d0336cc399363e 2023-09-06T20:15:46,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/c09b4b4418ef47169f08152710065395 is 50, key is test_row_0/C:col10/1694031345803/Put/seqid=0 2023-09-06T20:15:46,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/c09b4b4418ef47169f08152710065395 2023-09-06T20:15:46,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/32cf76982e2846b9ae2f0c8e7b9a484d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/32cf76982e2846b9ae2f0c8e7b9a484d 2023-09-06T20:15:46,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/32cf76982e2846b9ae2f0c8e7b9a484d, entries=200, sequenceid=402, filesize=14.4 K 2023-09-06T20:15:46,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/4abc4761130041b689d0336cc399363e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4abc4761130041b689d0336cc399363e 2023-09-06T20:15:46,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4abc4761130041b689d0336cc399363e, entries=150, sequenceid=402, filesize=12.0 K 2023-09-06T20:15:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/c09b4b4418ef47169f08152710065395 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/c09b4b4418ef47169f08152710065395 2023-09-06T20:15:46,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/c09b4b4418ef47169f08152710065395, entries=150, sequenceid=402, filesize=12.0 K 2023-09-06T20:15:46,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 969b36246b1411b7cf0e9c1debb5278b in 60ms, sequenceid=402, compaction requested=true 2023-09-06T20:15:46,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:46,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:46,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:46,996 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:46,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:46,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:46,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:46,996 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:46,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:46,997 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54426 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:46,997 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:46,997 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/A is initiating minor compaction (all files) 2023-09-06T20:15:46,997 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/B is initiating minor compaction (all files) 2023-09-06T20:15:46,997 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/A in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:46,998 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/B in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:46,998 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e981323ce62b42778f11683ea2258a49, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e8be59989dff44828f88f42f757f112b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/8a86b18679334b8c9782fffb9ecb6511, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/32cf76982e2846b9ae2f0c8e7b9a484d] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=53.2 K 2023-09-06T20:15:46,998 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/705bf7395879464ea53b0ca05cfcab8b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/10910ca76b0a49c18febbde87504902d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/3b311894be194ae3bd6d550d9b67b670, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4abc4761130041b689d0336cc399363e] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=48.4 K 2023-09-06T20:15:46,998 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 705bf7395879464ea53b0ca05cfcab8b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1694031344989 2023-09-06T20:15:46,998 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting e981323ce62b42778f11683ea2258a49, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1694031344989 2023-09-06T20:15:46,998 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 10910ca76b0a49c18febbde87504902d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1694031345031 2023-09-06T20:15:46,998 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting e8be59989dff44828f88f42f757f112b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1694031345031 2023-09-06T20:15:46,999 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 3b311894be194ae3bd6d550d9b67b670, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1694031345145 2023-09-06T20:15:46,999 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8a86b18679334b8c9782fffb9ecb6511, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1694031345145 2023-09-06T20:15:46,999 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 4abc4761130041b689d0336cc399363e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1694031345803 2023-09-06T20:15:46,999 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 32cf76982e2846b9ae2f0c8e7b9a484d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1694031345803 2023-09-06T20:15:47,007 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#A#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:47,008 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/cb5f7726bed04d028bfecd623b6ee21c is 50, key is test_row_0/A:col10/1694031345803/Put/seqid=0 2023-09-06T20:15:47,010 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#B#compaction#503 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:47,010 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/9979bf93abe0402a911111a0a30939f2 is 50, key is test_row_0/B:col10/1694031345803/Put/seqid=0 2023-09-06T20:15:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:47,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:15:47,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:47,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:47,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:47,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:47,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:47,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:47,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/926f800f99f741d08293d12d695feca0 is 50, key is test_row_0/A:col10/1694031347049/Put/seqid=0 2023-09-06T20:15:47,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/926f800f99f741d08293d12d695feca0 2023-09-06T20:15:47,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/56a3f986e1664bb6b6a632943d7d819c is 50, key is test_row_0/B:col10/1694031347049/Put/seqid=0 2023-09-06T20:15:47,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/56a3f986e1664bb6b6a632943d7d819c 2023-09-06T20:15:47,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/3b5952b30d2d49088f5620f62a96b736 is 50, key is test_row_0/C:col10/1694031347049/Put/seqid=0 2023-09-06T20:15:47,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031407083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031407083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031407086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031407090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031407091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-09-06T20:15:47,183 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 152 completed 2023-09-06T20:15:47,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2023-09-06T20:15:47,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=154 2023-09-06T20:15:47,187 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:47,187 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:47,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:47,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031407192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031407197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031407197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031407198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031407198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=154 2023-09-06T20:15:47,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2023-09-06T20:15:47,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:47,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:47,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:47,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(53): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:47,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:47,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031407399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031407403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031407403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031407403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031407403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,424 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/cb5f7726bed04d028bfecd623b6ee21c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/cb5f7726bed04d028bfecd623b6ee21c 2023-09-06T20:15:47,428 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/A of 969b36246b1411b7cf0e9c1debb5278b into cb5f7726bed04d028bfecd623b6ee21c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:47,428 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:47,428 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/A, priority=12, startTime=1694031346996; duration=0sec 2023-09-06T20:15:47,428 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:47,428 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:A 2023-09-06T20:15:47,428 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:47,431 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:47,431 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/C is initiating minor compaction (all files) 2023-09-06T20:15:47,431 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/C in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:47,431 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/025da8a1e54c46c8b56070f645e7f65f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/4361d0a71493482da9192982363684e7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5f4d810d23a942f6bd86ad800cfdcdf4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/c09b4b4418ef47169f08152710065395] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=48.4 K 2023-09-06T20:15:47,432 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 025da8a1e54c46c8b56070f645e7f65f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1694031344989 2023-09-06T20:15:47,432 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4361d0a71493482da9192982363684e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1694031345031 2023-09-06T20:15:47,433 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/9979bf93abe0402a911111a0a30939f2 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/9979bf93abe0402a911111a0a30939f2 2023-09-06T20:15:47,433 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5f4d810d23a942f6bd86ad800cfdcdf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1694031345145 2023-09-06T20:15:47,435 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting c09b4b4418ef47169f08152710065395, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1694031345803 2023-09-06T20:15:47,438 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/B of 969b36246b1411b7cf0e9c1debb5278b into 9979bf93abe0402a911111a0a30939f2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:47,438 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:47,438 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/B, priority=12, startTime=1694031346996; duration=0sec 2023-09-06T20:15:47,439 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:47,439 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:B 2023-09-06T20:15:47,441 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#C#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:47,442 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/f5faf9927d36420885601720ddbe7d58 is 50, key is test_row_0/C:col10/1694031345803/Put/seqid=0 2023-09-06T20:15:47,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/3b5952b30d2d49088f5620f62a96b736 2023-09-06T20:15:47,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/926f800f99f741d08293d12d695feca0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/926f800f99f741d08293d12d695feca0 2023-09-06T20:15:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=154 2023-09-06T20:15:47,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/926f800f99f741d08293d12d695feca0, entries=200, sequenceid=413, filesize=14.4 K 2023-09-06T20:15:47,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2023-09-06T20:15:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:47,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(53): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:47,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/56a3f986e1664bb6b6a632943d7d819c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/56a3f986e1664bb6b6a632943d7d819c 2023-09-06T20:15:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:15:47,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/56a3f986e1664bb6b6a632943d7d819c, entries=150, sequenceid=413, filesize=12.0 K 2023-09-06T20:15:47,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/3b5952b30d2d49088f5620f62a96b736 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/3b5952b30d2d49088f5620f62a96b736 2023-09-06T20:15:47,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/3b5952b30d2d49088f5620f62a96b736, entries=150, sequenceid=413, filesize=12.0 K 2023-09-06T20:15:47,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 969b36246b1411b7cf0e9c1debb5278b in 452ms, sequenceid=413, compaction requested=false 2023-09-06T20:15:47,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:47,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2023-09-06T20:15:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:47,646 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-06T20:15:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:47,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/952c27ef29054f0e88e10ea8717d979c is 50, key is test_row_0/A:col10/1694031347082/Put/seqid=0 2023-09-06T20:15:47,659 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/952c27ef29054f0e88e10ea8717d979c 2023-09-06T20:15:47,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/ef3b9b07fc2a406cb0369db2f3770782 is 50, key is test_row_0/B:col10/1694031347082/Put/seqid=0 2023-09-06T20:15:47,682 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/ef3b9b07fc2a406cb0369db2f3770782 2023-09-06T20:15:47,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/cb3bed0cc14c4b78a4d05d7f20eac783 is 50, key is test_row_0/C:col10/1694031347082/Put/seqid=0 2023-09-06T20:15:47,704 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/cb3bed0cc14c4b78a4d05d7f20eac783 2023-09-06T20:15:47,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/952c27ef29054f0e88e10ea8717d979c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/952c27ef29054f0e88e10ea8717d979c 2023-09-06T20:15:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:47,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:47,715 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/952c27ef29054f0e88e10ea8717d979c, entries=150, sequenceid=440, filesize=12.0 K 2023-09-06T20:15:47,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/ef3b9b07fc2a406cb0369db2f3770782 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/ef3b9b07fc2a406cb0369db2f3770782 2023-09-06T20:15:47,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031407713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031407714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031407715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031407718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,723 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/ef3b9b07fc2a406cb0369db2f3770782, entries=150, sequenceid=440, filesize=12.0 K 2023-09-06T20:15:47,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/cb3bed0cc14c4b78a4d05d7f20eac783 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/cb3bed0cc14c4b78a4d05d7f20eac783 2023-09-06T20:15:47,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031407720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,727 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/cb3bed0cc14c4b78a4d05d7f20eac783, entries=150, sequenceid=440, filesize=12.0 K 2023-09-06T20:15:47,728 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 969b36246b1411b7cf0e9c1debb5278b in 82ms, sequenceid=440, compaction requested=true 2023-09-06T20:15:47,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:47,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:47,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=155 2023-09-06T20:15:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=155 2023-09-06T20:15:47,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=155, resume processing ppid=154 2023-09-06T20:15:47,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=155, ppid=154, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 541 msec 2023-09-06T20:15:47,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=154, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 547 msec 2023-09-06T20:15:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=154 2023-09-06T20:15:47,790 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 154 completed 2023-09-06T20:15:47,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2023-09-06T20:15:47,795 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=156 2023-09-06T20:15:47,795 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:47,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:47,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:15:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:47,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/a2887eaa390942aa92e53635a5684c9b is 50, key is test_row_0/A:col10/1694031347824/Put/seqid=0 2023-09-06T20:15:47,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/a2887eaa390942aa92e53635a5684c9b 2023-09-06T20:15:47,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/6f35e04528a64e2484349b712b3dbede is 50, key is test_row_0/B:col10/1694031347824/Put/seqid=0 2023-09-06T20:15:47,866 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/f5faf9927d36420885601720ddbe7d58 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/f5faf9927d36420885601720ddbe7d58 2023-09-06T20:15:47,872 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/C of 969b36246b1411b7cf0e9c1debb5278b into f5faf9927d36420885601720ddbe7d58(size=12.5 K), total size for store is 36.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:47,872 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:47,872 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/C, priority=12, startTime=1694031346996; duration=0sec 2023-09-06T20:15:47,872 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:47,872 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:C 2023-09-06T20:15:47,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031407866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031407867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031407867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031407870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031407870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/6f35e04528a64e2484349b712b3dbede 2023-09-06T20:15:47,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/fbf7a6d07e334a41a192dd8c5212c79a is 50, key is test_row_0/C:col10/1694031347824/Put/seqid=0 2023-09-06T20:15:47,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/fbf7a6d07e334a41a192dd8c5212c79a 2023-09-06T20:15:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=156 2023-09-06T20:15:47,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/a2887eaa390942aa92e53635a5684c9b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/a2887eaa390942aa92e53635a5684c9b 2023-09-06T20:15:47,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/a2887eaa390942aa92e53635a5684c9b, entries=150, sequenceid=453, filesize=12.0 K 2023-09-06T20:15:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/6f35e04528a64e2484349b712b3dbede as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/6f35e04528a64e2484349b712b3dbede 2023-09-06T20:15:47,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/6f35e04528a64e2484349b712b3dbede, entries=100, sequenceid=453, filesize=9.6 K 2023-09-06T20:15:47,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/fbf7a6d07e334a41a192dd8c5212c79a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/fbf7a6d07e334a41a192dd8c5212c79a 2023-09-06T20:15:47,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/fbf7a6d07e334a41a192dd8c5212c79a, entries=100, sequenceid=453, filesize=9.6 K 2023-09-06T20:15:47,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 969b36246b1411b7cf0e9c1debb5278b in 88ms, sequenceid=453, compaction requested=true 2023-09-06T20:15:47,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:47,913 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:47,913 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:47,914 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52118 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:47,914 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47238 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:47,914 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/A is initiating minor compaction (all files) 2023-09-06T20:15:47,914 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/B is initiating minor compaction (all files) 2023-09-06T20:15:47,914 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/A in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:47,914 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/B in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:47,914 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/cb5f7726bed04d028bfecd623b6ee21c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/926f800f99f741d08293d12d695feca0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/952c27ef29054f0e88e10ea8717d979c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/a2887eaa390942aa92e53635a5684c9b] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=50.9 K 2023-09-06T20:15:47,914 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/9979bf93abe0402a911111a0a30939f2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/56a3f986e1664bb6b6a632943d7d819c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/ef3b9b07fc2a406cb0369db2f3770782, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/6f35e04528a64e2484349b712b3dbede] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=46.1 K 2023-09-06T20:15:47,914 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting cb5f7726bed04d028bfecd623b6ee21c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1694031345803 2023-09-06T20:15:47,914 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 9979bf93abe0402a911111a0a30939f2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1694031345803 2023-09-06T20:15:47,915 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 926f800f99f741d08293d12d695feca0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1694031346941 2023-09-06T20:15:47,915 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 56a3f986e1664bb6b6a632943d7d819c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1694031346941 2023-09-06T20:15:47,915 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 952c27ef29054f0e88e10ea8717d979c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1694031347082 2023-09-06T20:15:47,915 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting ef3b9b07fc2a406cb0369db2f3770782, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1694031347082 2023-09-06T20:15:47,915 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting a2887eaa390942aa92e53635a5684c9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1694031347718 2023-09-06T20:15:47,915 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 6f35e04528a64e2484349b712b3dbede, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1694031347823 2023-09-06T20:15:47,931 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#A#compaction#514 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:47,931 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/c0a0923d1370439fb0f94695440f9542 is 50, key is test_row_0/A:col10/1694031347824/Put/seqid=0 2023-09-06T20:15:47,933 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#B#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:47,934 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/4b5a6971a13a4a05a8814efe80b32a18 is 50, key is test_row_0/B:col10/1694031347824/Put/seqid=0 2023-09-06T20:15:47,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2023-09-06T20:15:47,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:47,949 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-06T20:15:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:47,951 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/c0a0923d1370439fb0f94695440f9542 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/c0a0923d1370439fb0f94695440f9542 2023-09-06T20:15:47,955 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/A of 969b36246b1411b7cf0e9c1debb5278b into c0a0923d1370439fb0f94695440f9542(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:47,955 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:47,955 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/A, priority=12, startTime=1694031347912; duration=0sec 2023-09-06T20:15:47,955 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:47,955 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:A 2023-09-06T20:15:47,955 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:15:47,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/c4ce02b3bfb348ccbf44bd2bae11d3d6 is 50, key is test_row_0/A:col10/1694031347867/Put/seqid=0 2023-09-06T20:15:47,958 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47238 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:15:47,959 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/C is initiating minor compaction (all files) 2023-09-06T20:15:47,959 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/C in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:47,959 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/f5faf9927d36420885601720ddbe7d58, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/3b5952b30d2d49088f5620f62a96b736, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/cb3bed0cc14c4b78a4d05d7f20eac783, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/fbf7a6d07e334a41a192dd8c5212c79a] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=46.1 K 2023-09-06T20:15:47,959 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting f5faf9927d36420885601720ddbe7d58, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1694031345803 2023-09-06T20:15:47,960 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3b5952b30d2d49088f5620f62a96b736, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1694031346941 2023-09-06T20:15:47,960 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting cb3bed0cc14c4b78a4d05d7f20eac783, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1694031347082 2023-09-06T20:15:47,961 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting fbf7a6d07e334a41a192dd8c5212c79a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1694031347823 2023-09-06T20:15:47,962 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/4b5a6971a13a4a05a8814efe80b32a18 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4b5a6971a13a4a05a8814efe80b32a18 2023-09-06T20:15:47,970 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/B of 969b36246b1411b7cf0e9c1debb5278b into 4b5a6971a13a4a05a8814efe80b32a18(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:47,970 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:47,970 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/B, priority=12, startTime=1694031347913; duration=0sec 2023-09-06T20:15:47,970 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:47,970 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:B 2023-09-06T20:15:47,972 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#C#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:47,972 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/f7a4dab53c964d42ad58baf90b2da106 is 50, key is test_row_0/C:col10/1694031347824/Put/seqid=0 2023-09-06T20:15:47,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:47,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031407984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031407988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031407989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031407989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:47,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031407991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,000 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/f7a4dab53c964d42ad58baf90b2da106 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/f7a4dab53c964d42ad58baf90b2da106 2023-09-06T20:15:48,003 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/C of 969b36246b1411b7cf0e9c1debb5278b into f7a4dab53c964d42ad58baf90b2da106(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:48,004 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:48,004 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/C, priority=12, startTime=1694031347913; duration=0sec 2023-09-06T20:15:48,004 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:48,004 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:C 2023-09-06T20:15:48,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031408091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031408091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=156 2023-09-06T20:15:48,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031408096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031408096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031408096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031408294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031408296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031408300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031408301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031408301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,374 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/c4ce02b3bfb348ccbf44bd2bae11d3d6 2023-09-06T20:15:48,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/8d39b3cbb7404447addf74b236ea3573 is 50, key is test_row_0/B:col10/1694031347867/Put/seqid=0 2023-09-06T20:15:48,388 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/8d39b3cbb7404447addf74b236ea3573 2023-09-06T20:15:48,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/612ec5136433423b87b8b9ae20a70295 is 50, key is test_row_0/C:col10/1694031347867/Put/seqid=0 2023-09-06T20:15:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=156 2023-09-06T20:15:48,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031408598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031408604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031408604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031408607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031408607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:48,804 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/612ec5136433423b87b8b9ae20a70295 2023-09-06T20:15:48,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/c4ce02b3bfb348ccbf44bd2bae11d3d6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/c4ce02b3bfb348ccbf44bd2bae11d3d6 2023-09-06T20:15:48,817 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/c4ce02b3bfb348ccbf44bd2bae11d3d6, entries=150, sequenceid=478, filesize=12.0 K 2023-09-06T20:15:48,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/8d39b3cbb7404447addf74b236ea3573 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8d39b3cbb7404447addf74b236ea3573 2023-09-06T20:15:48,821 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8d39b3cbb7404447addf74b236ea3573, entries=150, sequenceid=478, filesize=12.0 K 2023-09-06T20:15:48,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/612ec5136433423b87b8b9ae20a70295 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/612ec5136433423b87b8b9ae20a70295 2023-09-06T20:15:48,828 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/612ec5136433423b87b8b9ae20a70295, entries=150, sequenceid=478, filesize=12.0 K 2023-09-06T20:15:48,828 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 969b36246b1411b7cf0e9c1debb5278b in 880ms, sequenceid=478, compaction requested=false 2023-09-06T20:15:48,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:48,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:48,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=157 2023-09-06T20:15:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=157 2023-09-06T20:15:48,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=157, resume processing ppid=156 2023-09-06T20:15:48,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=157, ppid=156, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0330 sec 2023-09-06T20:15:48,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=156, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0390 sec 2023-09-06T20:15:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=156 2023-09-06T20:15:48,898 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 156 completed 2023-09-06T20:15:48,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2023-09-06T20:15:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=158 2023-09-06T20:15:48,907 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:48,908 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:48,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=158 2023-09-06T20:15:49,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2023-09-06T20:15:49,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:49,062 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-06T20:15:49,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:49,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:49,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:49,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:49,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:49,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:49,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/638e6e0305564344b449722cb646b19f is 50, key is test_row_0/A:col10/1694031347986/Put/seqid=0 2023-09-06T20:15:49,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:49,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031409141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031409141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031409142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031409144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031409144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=158 2023-09-06T20:15:49,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031409248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031409248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031409248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031409252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031409252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031409453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031409454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031409455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031409458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031409459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,476 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/638e6e0305564344b449722cb646b19f 2023-09-06T20:15:49,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/fb776d706b8149f8bfc20035aa17d79c is 50, key is test_row_0/B:col10/1694031347986/Put/seqid=0 2023-09-06T20:15:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=158 2023-09-06T20:15:49,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031409760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031409761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031409761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031409765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031409767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:49,891 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/fb776d706b8149f8bfc20035aa17d79c 2023-09-06T20:15:49,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5beb593455074516b7de0c081dfa6b68 is 50, key is test_row_0/C:col10/1694031347986/Put/seqid=0 2023-09-06T20:15:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=158 2023-09-06T20:15:50,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:50,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031410265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:50,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:50,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031410266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:50,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031410269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:50,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031410271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:50,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031410273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:50,312 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5beb593455074516b7de0c081dfa6b68 2023-09-06T20:15:50,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/638e6e0305564344b449722cb646b19f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/638e6e0305564344b449722cb646b19f 2023-09-06T20:15:50,320 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/638e6e0305564344b449722cb646b19f, entries=150, sequenceid=493, filesize=12.0 K 2023-09-06T20:15:50,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/fb776d706b8149f8bfc20035aa17d79c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/fb776d706b8149f8bfc20035aa17d79c 2023-09-06T20:15:50,324 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/fb776d706b8149f8bfc20035aa17d79c, entries=150, sequenceid=493, filesize=12.0 K 2023-09-06T20:15:50,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5beb593455074516b7de0c081dfa6b68 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5beb593455074516b7de0c081dfa6b68 2023-09-06T20:15:50,328 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5beb593455074516b7de0c081dfa6b68, entries=150, sequenceid=493, filesize=12.0 K 2023-09-06T20:15:50,329 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 969b36246b1411b7cf0e9c1debb5278b in 1267ms, sequenceid=493, compaction requested=true 2023-09-06T20:15:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=159 2023-09-06T20:15:50,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=159 2023-09-06T20:15:50,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=159, resume processing ppid=158 2023-09-06T20:15:50,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=159, ppid=158, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4220 sec 2023-09-06T20:15:50,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=158, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.4320 sec 2023-09-06T20:15:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=158 2023-09-06T20:15:51,012 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 158 completed 2023-09-06T20:15:51,014 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2023-09-06T20:15:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=160 2023-09-06T20:15:51,016 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:51,017 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:51,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=160 2023-09-06T20:15:51,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2023-09-06T20:15:51,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:51,170 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-06T20:15:51,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:51,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:51,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:51,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:51,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:51,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:51,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/df98f41261304a5781b210a5125b873f is 50, key is test_row_0/A:col10/1694031349141/Put/seqid=0 2023-09-06T20:15:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:51,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. as already flushing 2023-09-06T20:15:51,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031411284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:51,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031411284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:51,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42990 deadline: 1694031411284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:51,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031411289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:51,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031411290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=160 2023-09-06T20:15:51,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031411393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:51,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031411393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:51,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031411396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:51,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031411396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:51,581 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/df98f41261304a5781b210a5125b873f 2023-09-06T20:15:51,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/e6b83a21313c4b61b464b5a4ecd975c7 is 50, key is test_row_0/B:col10/1694031349141/Put/seqid=0 2023-09-06T20:15:51,596 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/e6b83a21313c4b61b464b5a4ecd975c7 2023-09-06T20:15:51,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5c2512d6ddc54291b1abdc9d80d2041e is 50, key is test_row_0/C:col10/1694031349141/Put/seqid=0 2023-09-06T20:15:51,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43028 deadline: 1694031411599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:51,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42980 deadline: 1694031411599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:51,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:43018 deadline: 1694031411601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:51,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42976 deadline: 1694031411602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:51,611 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5c2512d6ddc54291b1abdc9d80d2041e 2023-09-06T20:15:51,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/df98f41261304a5781b210a5125b873f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/df98f41261304a5781b210a5125b873f 2023-09-06T20:15:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=160 2023-09-06T20:15:51,622 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/df98f41261304a5781b210a5125b873f, entries=150, sequenceid=518, filesize=12.0 K 2023-09-06T20:15:51,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/e6b83a21313c4b61b464b5a4ecd975c7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/e6b83a21313c4b61b464b5a4ecd975c7 2023-09-06T20:15:51,628 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/e6b83a21313c4b61b464b5a4ecd975c7, entries=150, sequenceid=518, filesize=12.0 K 2023-09-06T20:15:51,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/5c2512d6ddc54291b1abdc9d80d2041e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5c2512d6ddc54291b1abdc9d80d2041e 2023-09-06T20:15:51,633 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5c2512d6ddc54291b1abdc9d80d2041e, entries=150, sequenceid=518, filesize=12.0 K 2023-09-06T20:15:51,634 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 969b36246b1411b7cf0e9c1debb5278b in 464ms, sequenceid=518, compaction requested=true 2023-09-06T20:15:51,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:51,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:51,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=161 2023-09-06T20:15:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=161 2023-09-06T20:15:51,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=161, resume processing ppid=160 2023-09-06T20:15:51,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=161, ppid=160, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 618 msec 2023-09-06T20:15:51,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=160, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 623 msec 2023-09-06T20:15:51,788 DEBUG [Thread-2642 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x560b4319 to 127.0.0.1:58481 2023-09-06T20:15:51,788 DEBUG [Thread-2642 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:51,789 DEBUG [Thread-2644 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x233f731e to 127.0.0.1:58481 2023-09-06T20:15:51,789 DEBUG [Thread-2644 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:51,791 DEBUG [Thread-2646 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x258a8814 to 127.0.0.1:58481 2023-09-06T20:15:51,791 DEBUG [Thread-2646 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:51,791 DEBUG [Thread-2640 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0f6aeb1b to 127.0.0.1:58481 2023-09-06T20:15:51,791 DEBUG [Thread-2640 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:51,793 DEBUG [Thread-2638 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x06f6e6dc to 127.0.0.1:58481 2023-09-06T20:15:51,793 DEBUG [Thread-2638 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:51,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-06T20:15:51,908 DEBUG [Thread-2633 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x70c70c97 to 127.0.0.1:58481 2023-09-06T20:15:51,908 DEBUG [Thread-2633 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:51,908 DEBUG [Thread-2631 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4f8d201b to 127.0.0.1:58481 2023-09-06T20:15:51,908 DEBUG [Thread-2631 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:51,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:51,928 DEBUG [Thread-2629 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x768de042 to 127.0.0.1:58481 2023-09-06T20:15:51,928 DEBUG [Thread-2627 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x05908814 to 127.0.0.1:58481 2023-09-06T20:15:51,928 DEBUG [Thread-2627 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:51,928 DEBUG [Thread-2629 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:51,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:51,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:51,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:51,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:51,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:51,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/063d326f0cef4fb1a5cfa55262ced04c is 50, key is test_row_0/A:col10/1694031351907/Put/seqid=0 2023-09-06T20:15:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=160 2023-09-06T20:15:52,120 INFO [Thread-2637 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 160 completed 2023-09-06T20:15:52,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/063d326f0cef4fb1a5cfa55262ced04c 2023-09-06T20:15:52,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/c1b79dc6147e421ba0d01f1a2bbd06ba is 50, key is test_row_0/B:col10/1694031351907/Put/seqid=0 2023-09-06T20:15:52,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/c1b79dc6147e421ba0d01f1a2bbd06ba 2023-09-06T20:15:52,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/07082a34048b4744892478f12e2c2973 is 50, key is test_row_0/C:col10/1694031351907/Put/seqid=0 2023-09-06T20:15:53,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/07082a34048b4744892478f12e2c2973 2023-09-06T20:15:53,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/063d326f0cef4fb1a5cfa55262ced04c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/063d326f0cef4fb1a5cfa55262ced04c 2023-09-06T20:15:53,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/063d326f0cef4fb1a5cfa55262ced04c, entries=150, sequenceid=533, filesize=12.0 K 2023-09-06T20:15:53,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/c1b79dc6147e421ba0d01f1a2bbd06ba as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/c1b79dc6147e421ba0d01f1a2bbd06ba 2023-09-06T20:15:53,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/c1b79dc6147e421ba0d01f1a2bbd06ba, entries=150, sequenceid=533, filesize=12.0 K 2023-09-06T20:15:53,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/07082a34048b4744892478f12e2c2973 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/07082a34048b4744892478f12e2c2973 2023-09-06T20:15:53,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/07082a34048b4744892478f12e2c2973, entries=150, sequenceid=533, filesize=12.0 K 2023-09-06T20:15:53,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for 969b36246b1411b7cf0e9c1debb5278b in 1259ms, sequenceid=533, compaction requested=true 2023-09-06T20:15:53,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:53,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:15:53,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:53,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:15:53,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:53,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969b36246b1411b7cf0e9c1debb5278b:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:15:53,168 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2023-09-06T20:15:53,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:53,168 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2023-09-06T20:15:53,169 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62102 starting at candidate #0 after considering 6 permutations with 6 in ratio 2023-09-06T20:15:53,169 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/B is initiating minor compaction (all files) 2023-09-06T20:15:53,169 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62102 starting at candidate #0 after considering 6 permutations with 6 in ratio 2023-09-06T20:15:53,169 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/B in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:53,169 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 969b36246b1411b7cf0e9c1debb5278b/A is initiating minor compaction (all files) 2023-09-06T20:15:53,169 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4b5a6971a13a4a05a8814efe80b32a18, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8d39b3cbb7404447addf74b236ea3573, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/fb776d706b8149f8bfc20035aa17d79c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/e6b83a21313c4b61b464b5a4ecd975c7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/c1b79dc6147e421ba0d01f1a2bbd06ba] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=60.6 K 2023-09-06T20:15:53,169 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 969b36246b1411b7cf0e9c1debb5278b/A in TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:53,169 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/c0a0923d1370439fb0f94695440f9542, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/c4ce02b3bfb348ccbf44bd2bae11d3d6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/638e6e0305564344b449722cb646b19f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/df98f41261304a5781b210a5125b873f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/063d326f0cef4fb1a5cfa55262ced04c] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp, totalSize=60.6 K 2023-09-06T20:15:53,170 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 4b5a6971a13a4a05a8814efe80b32a18, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1694031347089 2023-09-06T20:15:53,170 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting c0a0923d1370439fb0f94695440f9542, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1694031347089 2023-09-06T20:15:53,170 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 8d39b3cbb7404447addf74b236ea3573, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1694031347863 2023-09-06T20:15:53,170 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting c4ce02b3bfb348ccbf44bd2bae11d3d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1694031347863 2023-09-06T20:15:53,170 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting fb776d706b8149f8bfc20035aa17d79c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1694031347986 2023-09-06T20:15:53,170 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 638e6e0305564344b449722cb646b19f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1694031347986 2023-09-06T20:15:53,170 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting e6b83a21313c4b61b464b5a4ecd975c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1694031349141 2023-09-06T20:15:53,171 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting df98f41261304a5781b210a5125b873f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1694031349141 2023-09-06T20:15:53,171 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting c1b79dc6147e421ba0d01f1a2bbd06ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1694031351289 2023-09-06T20:15:53,171 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 063d326f0cef4fb1a5cfa55262ced04c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1694031351289 2023-09-06T20:15:53,178 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#A#compaction#529 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:53,179 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969b36246b1411b7cf0e9c1debb5278b#B#compaction#530 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:15:53,179 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/bf813ee285b344de88682044b06f2975 is 50, key is test_row_0/A:col10/1694031351907/Put/seqid=0 2023-09-06T20:15:53,179 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/1e1a265e22e54b8ba9ed7aafdb01b370 is 50, key is test_row_0/B:col10/1694031351907/Put/seqid=0 2023-09-06T20:15:53,309 DEBUG [Thread-2635 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x319d6df7 to 127.0.0.1:58481 2023-09-06T20:15:53,309 DEBUG [Thread-2635 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:53,309 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2023-09-06T20:15:53,309 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2023-09-06T20:15:53,309 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2023-09-06T20:15:53,309 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2023-09-06T20:15:53,309 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2023-09-06T20:15:53,309 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2023-09-06T20:15:53,309 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2023-09-06T20:15:53,309 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2023-09-06T20:15:53,310 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(402): scanned 2482 2023-09-06T20:15:53,310 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(403): verified 7446 rows 2023-09-06T20:15:53,310 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(402): scanned 2494 2023-09-06T20:15:53,310 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(403): verified 7481 rows 2023-09-06T20:15:53,310 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(402): scanned 2490 2023-09-06T20:15:53,310 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(403): verified 7470 rows 2023-09-06T20:15:53,310 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(402): scanned 2503 2023-09-06T20:15:53,310 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(403): verified 7509 rows 2023-09-06T20:15:53,310 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(402): scanned 2491 2023-09-06T20:15:53,310 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(403): verified 7472 rows 2023-09-06T20:15:53,310 INFO [Listener at localhost/35961 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:15:53,310 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x287d13f5 to 127.0.0.1:58481 2023-09-06T20:15:53,310 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:15:53,312 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2023-09-06T20:15:53,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable TestAcidGuarantees 2023-09-06T20:15:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=162 2023-09-06T20:15:53,317 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031353317"}]},"ts":"1694031353317"} 2023-09-06T20:15:53,318 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2023-09-06T20:15:53,320 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2023-09-06T20:15:53,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=969b36246b1411b7cf0e9c1debb5278b, UNASSIGN}] 2023-09-06T20:15:53,322 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=969b36246b1411b7cf0e9c1debb5278b, UNASSIGN 2023-09-06T20:15:53,322 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=163 updating hbase:meta row=969b36246b1411b7cf0e9c1debb5278b, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:53,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE; CloseRegionProcedure 969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:15:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=162 2023-09-06T20:15:53,474 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] handler.UnassignRegionHandler(119): Close 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:53,475 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1668): Closing 969b36246b1411b7cf0e9c1debb5278b, disabling compactions & flushes 2023-09-06T20:15:53,475 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1929): waiting for 2 compactions to complete for region TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:53,593 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/bf813ee285b344de88682044b06f2975 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bf813ee285b344de88682044b06f2975 2023-09-06T20:15:53,593 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/1e1a265e22e54b8ba9ed7aafdb01b370 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/1e1a265e22e54b8ba9ed7aafdb01b370 2023-09-06T20:15:53,596 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 5 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/A of 969b36246b1411b7cf0e9c1debb5278b into bf813ee285b344de88682044b06f2975(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:53,596 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:53,596 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/A, priority=11, startTime=1694031353168; duration=0sec 2023-09-06T20:15:53,596 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:15:53,596 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:A 2023-09-06T20:15:53,596 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. because compaction request was cancelled 2023-09-06T20:15:53,596 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:C 2023-09-06T20:15:53,596 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 5 (all) file(s) in 969b36246b1411b7cf0e9c1debb5278b/B of 969b36246b1411b7cf0e9c1debb5278b into 1e1a265e22e54b8ba9ed7aafdb01b370(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:15:53,596 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:53,596 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b., storeName=969b36246b1411b7cf0e9c1debb5278b/B, priority=11, startTime=1694031353168; duration=0sec 2023-09-06T20:15:53,596 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:53,596 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:53,596 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:15:53,597 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. after waiting 0 ms 2023-09-06T20:15:53,597 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969b36246b1411b7cf0e9c1debb5278b:B 2023-09-06T20:15:53,597 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:53,597 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(2819): Flushing 969b36246b1411b7cf0e9c1debb5278b 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2023-09-06T20:15:53,597 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=A 2023-09-06T20:15:53,597 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:53,597 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=B 2023-09-06T20:15:53,597 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:53,597 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 969b36246b1411b7cf0e9c1debb5278b, store=C 2023-09-06T20:15:53,597 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:53,599 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/1b584078b1614222898e77d305d523da is 50, key is test_row_1/A:col10/1694031353308/Put/seqid=0 2023-09-06T20:15:53,603 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/1b584078b1614222898e77d305d523da 2023-09-06T20:15:53,607 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/436553133a5d454fa34ee2b022df017b is 50, key is test_row_1/B:col10/1694031353308/Put/seqid=0 2023-09-06T20:15:53,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=162 2023-09-06T20:15:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=162 2023-09-06T20:15:54,011 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/436553133a5d454fa34ee2b022df017b 2023-09-06T20:15:54,018 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/22e25336e02f49168930c0a8d94b3e88 is 50, key is test_row_1/C:col10/1694031353308/Put/seqid=0 2023-09-06T20:15:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=162 2023-09-06T20:15:54,423 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/22e25336e02f49168930c0a8d94b3e88 2023-09-06T20:15:54,427 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/A/1b584078b1614222898e77d305d523da as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/1b584078b1614222898e77d305d523da 2023-09-06T20:15:54,430 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/1b584078b1614222898e77d305d523da, entries=50, sequenceid=539, filesize=7.2 K 2023-09-06T20:15:54,430 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/B/436553133a5d454fa34ee2b022df017b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/436553133a5d454fa34ee2b022df017b 2023-09-06T20:15:54,433 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/436553133a5d454fa34ee2b022df017b, entries=50, sequenceid=539, filesize=7.2 K 2023-09-06T20:15:54,434 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/.tmp/C/22e25336e02f49168930c0a8d94b3e88 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/22e25336e02f49168930c0a8d94b3e88 2023-09-06T20:15:54,437 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/22e25336e02f49168930c0a8d94b3e88, entries=50, sequenceid=539, filesize=7.2 K 2023-09-06T20:15:54,438 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(3022): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 969b36246b1411b7cf0e9c1debb5278b in 841ms, sequenceid=539, compaction requested=true 2023-09-06T20:15:54,439 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/34edbd77ac6142dd920468d9e94af21a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/cc887c10f85b40be89b0571111fb8903, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/b462f22c06df4c19bdcda6c63addb031, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/f235a65e37c84a11908eba892cbf1335, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/12501d23d4564b169c1bd28b7540e2b4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/4ce70c85c46d4646b740c1c039248fc7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/1c0fcb18f2564556b6ded88ab21cc2e8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e23611564d964a71aee8d00037e0c4c6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e5d5267801984573bad28502bb6d635e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/0335297fb4a741e7aec925eee93a707d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/5dec7d14f9b84e2faf6030a60b34cc7c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bedb9b281f924e259941bb4c4c32cee2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e981323ce62b42778f11683ea2258a49, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e8be59989dff44828f88f42f757f112b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/8a86b18679334b8c9782fffb9ecb6511, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/32cf76982e2846b9ae2f0c8e7b9a484d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/cb5f7726bed04d028bfecd623b6ee21c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/926f800f99f741d08293d12d695feca0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/952c27ef29054f0e88e10ea8717d979c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/c0a0923d1370439fb0f94695440f9542, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/a2887eaa390942aa92e53635a5684c9b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/c4ce02b3bfb348ccbf44bd2bae11d3d6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/638e6e0305564344b449722cb646b19f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/df98f41261304a5781b210a5125b873f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/063d326f0cef4fb1a5cfa55262ced04c] to archive 2023-09-06T20:15:54,440 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:15:54,441 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/34edbd77ac6142dd920468d9e94af21a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/34edbd77ac6142dd920468d9e94af21a 2023-09-06T20:15:54,442 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/cc887c10f85b40be89b0571111fb8903 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/cc887c10f85b40be89b0571111fb8903 2023-09-06T20:15:54,443 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/b462f22c06df4c19bdcda6c63addb031 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/b462f22c06df4c19bdcda6c63addb031 2023-09-06T20:15:54,444 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/f235a65e37c84a11908eba892cbf1335 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/f235a65e37c84a11908eba892cbf1335 2023-09-06T20:15:54,445 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/12501d23d4564b169c1bd28b7540e2b4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/12501d23d4564b169c1bd28b7540e2b4 2023-09-06T20:15:54,446 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/4ce70c85c46d4646b740c1c039248fc7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/4ce70c85c46d4646b740c1c039248fc7 2023-09-06T20:15:54,447 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/1c0fcb18f2564556b6ded88ab21cc2e8 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/1c0fcb18f2564556b6ded88ab21cc2e8 2023-09-06T20:15:54,448 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e23611564d964a71aee8d00037e0c4c6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e23611564d964a71aee8d00037e0c4c6 2023-09-06T20:15:54,449 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e5d5267801984573bad28502bb6d635e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e5d5267801984573bad28502bb6d635e 2023-09-06T20:15:54,450 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/0335297fb4a741e7aec925eee93a707d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/0335297fb4a741e7aec925eee93a707d 2023-09-06T20:15:54,451 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/5dec7d14f9b84e2faf6030a60b34cc7c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/5dec7d14f9b84e2faf6030a60b34cc7c 2023-09-06T20:15:54,452 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bedb9b281f924e259941bb4c4c32cee2 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bedb9b281f924e259941bb4c4c32cee2 2023-09-06T20:15:54,454 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e981323ce62b42778f11683ea2258a49 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e981323ce62b42778f11683ea2258a49 2023-09-06T20:15:54,455 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e8be59989dff44828f88f42f757f112b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/e8be59989dff44828f88f42f757f112b 2023-09-06T20:15:54,456 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/8a86b18679334b8c9782fffb9ecb6511 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/8a86b18679334b8c9782fffb9ecb6511 2023-09-06T20:15:54,457 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/32cf76982e2846b9ae2f0c8e7b9a484d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/32cf76982e2846b9ae2f0c8e7b9a484d 2023-09-06T20:15:54,458 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/cb5f7726bed04d028bfecd623b6ee21c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/cb5f7726bed04d028bfecd623b6ee21c 2023-09-06T20:15:54,459 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/926f800f99f741d08293d12d695feca0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/926f800f99f741d08293d12d695feca0 2023-09-06T20:15:54,460 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/952c27ef29054f0e88e10ea8717d979c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/952c27ef29054f0e88e10ea8717d979c 2023-09-06T20:15:54,464 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/c0a0923d1370439fb0f94695440f9542 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/c0a0923d1370439fb0f94695440f9542 2023-09-06T20:15:54,465 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/a2887eaa390942aa92e53635a5684c9b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/a2887eaa390942aa92e53635a5684c9b 2023-09-06T20:15:54,466 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/c4ce02b3bfb348ccbf44bd2bae11d3d6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/c4ce02b3bfb348ccbf44bd2bae11d3d6 2023-09-06T20:15:54,467 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/638e6e0305564344b449722cb646b19f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/638e6e0305564344b449722cb646b19f 2023-09-06T20:15:54,468 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/df98f41261304a5781b210a5125b873f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/df98f41261304a5781b210a5125b873f 2023-09-06T20:15:54,469 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/063d326f0cef4fb1a5cfa55262ced04c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/063d326f0cef4fb1a5cfa55262ced04c 2023-09-06T20:15:54,471 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/eda06905d26f4fdda6348f321e0c08c7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/1706ca118121493d8d3b64da1602925f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/3826c9b1f9124ec6b5f4b51f17b6cffa, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/5d4f49e3e389417dbf531ab0e71db939, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8450b64ad80643b78c307f8ddfb5b458, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/ae08c4d4b0014692b146beda0501d5b9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/503f8413d2234464afe18b56f1348a10, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/c4aa6a1bd3d24d8b87ca3732f5ebff05, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/f411f22ec5f245889508c5465764e993, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/455a82c1b9794959a18f92e71b5b2134, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/98475f353a0741e0bed1e62c7c0aa25a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/705bf7395879464ea53b0ca05cfcab8b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4354bd496d2a4e59b1d55624f32ee55d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/10910ca76b0a49c18febbde87504902d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/3b311894be194ae3bd6d550d9b67b670, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/9979bf93abe0402a911111a0a30939f2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4abc4761130041b689d0336cc399363e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/56a3f986e1664bb6b6a632943d7d819c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/ef3b9b07fc2a406cb0369db2f3770782, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4b5a6971a13a4a05a8814efe80b32a18, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/6f35e04528a64e2484349b712b3dbede, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8d39b3cbb7404447addf74b236ea3573, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/fb776d706b8149f8bfc20035aa17d79c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/e6b83a21313c4b61b464b5a4ecd975c7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/c1b79dc6147e421ba0d01f1a2bbd06ba] to archive 2023-09-06T20:15:54,471 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:15:54,473 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/eda06905d26f4fdda6348f321e0c08c7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/eda06905d26f4fdda6348f321e0c08c7 2023-09-06T20:15:54,474 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/1706ca118121493d8d3b64da1602925f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/1706ca118121493d8d3b64da1602925f 2023-09-06T20:15:54,475 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/3826c9b1f9124ec6b5f4b51f17b6cffa to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/3826c9b1f9124ec6b5f4b51f17b6cffa 2023-09-06T20:15:54,476 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/5d4f49e3e389417dbf531ab0e71db939 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/5d4f49e3e389417dbf531ab0e71db939 2023-09-06T20:15:54,477 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8450b64ad80643b78c307f8ddfb5b458 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8450b64ad80643b78c307f8ddfb5b458 2023-09-06T20:15:54,478 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/ae08c4d4b0014692b146beda0501d5b9 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/ae08c4d4b0014692b146beda0501d5b9 2023-09-06T20:15:54,479 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/503f8413d2234464afe18b56f1348a10 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/503f8413d2234464afe18b56f1348a10 2023-09-06T20:15:54,480 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/c4aa6a1bd3d24d8b87ca3732f5ebff05 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/c4aa6a1bd3d24d8b87ca3732f5ebff05 2023-09-06T20:15:54,481 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/f411f22ec5f245889508c5465764e993 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/f411f22ec5f245889508c5465764e993 2023-09-06T20:15:54,482 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/455a82c1b9794959a18f92e71b5b2134 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/455a82c1b9794959a18f92e71b5b2134 2023-09-06T20:15:54,483 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/98475f353a0741e0bed1e62c7c0aa25a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/98475f353a0741e0bed1e62c7c0aa25a 2023-09-06T20:15:54,484 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/705bf7395879464ea53b0ca05cfcab8b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/705bf7395879464ea53b0ca05cfcab8b 2023-09-06T20:15:54,485 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4354bd496d2a4e59b1d55624f32ee55d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4354bd496d2a4e59b1d55624f32ee55d 2023-09-06T20:15:54,486 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/10910ca76b0a49c18febbde87504902d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/10910ca76b0a49c18febbde87504902d 2023-09-06T20:15:54,487 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/3b311894be194ae3bd6d550d9b67b670 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/3b311894be194ae3bd6d550d9b67b670 2023-09-06T20:15:54,488 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/9979bf93abe0402a911111a0a30939f2 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/9979bf93abe0402a911111a0a30939f2 2023-09-06T20:15:54,489 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4abc4761130041b689d0336cc399363e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4abc4761130041b689d0336cc399363e 2023-09-06T20:15:54,490 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/56a3f986e1664bb6b6a632943d7d819c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/56a3f986e1664bb6b6a632943d7d819c 2023-09-06T20:15:54,491 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/ef3b9b07fc2a406cb0369db2f3770782 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/ef3b9b07fc2a406cb0369db2f3770782 2023-09-06T20:15:54,492 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4b5a6971a13a4a05a8814efe80b32a18 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/4b5a6971a13a4a05a8814efe80b32a18 2023-09-06T20:15:54,492 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/6f35e04528a64e2484349b712b3dbede to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/6f35e04528a64e2484349b712b3dbede 2023-09-06T20:15:54,493 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8d39b3cbb7404447addf74b236ea3573 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/8d39b3cbb7404447addf74b236ea3573 2023-09-06T20:15:54,494 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/fb776d706b8149f8bfc20035aa17d79c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/fb776d706b8149f8bfc20035aa17d79c 2023-09-06T20:15:54,495 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/e6b83a21313c4b61b464b5a4ecd975c7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/e6b83a21313c4b61b464b5a4ecd975c7 2023-09-06T20:15:54,496 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/c1b79dc6147e421ba0d01f1a2bbd06ba to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/c1b79dc6147e421ba0d01f1a2bbd06ba 2023-09-06T20:15:54,497 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5d57636bfe1d4f159972e307b570ed3e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/67b1e7fc7b334f8db5a99d50583ce0a0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/b14a23c51e2e4001bfde2f5e416fbd3e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a12abad821ad4e4cbc7173181963d6ce, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/188b53f56434403b9c7c7a0406b203e8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/ca9024745bd5403b8710cd6eaac37a9a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/bdd4f68767094283b1e47be11f4576b9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/73451464d4c14568b3a400f77974f09a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5dfa56afa1b94a63a781e5f87498788c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/48741bacb05b488c8788c928c6e40ebb, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5f38c0005a744187a593f2a6156d25de, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/025da8a1e54c46c8b56070f645e7f65f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/538af06c9b064bb4bb9ad4cc245accf3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/4361d0a71493482da9192982363684e7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5f4d810d23a942f6bd86ad800cfdcdf4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/f5faf9927d36420885601720ddbe7d58, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/c09b4b4418ef47169f08152710065395, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/3b5952b30d2d49088f5620f62a96b736, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/cb3bed0cc14c4b78a4d05d7f20eac783, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/fbf7a6d07e334a41a192dd8c5212c79a] to archive 2023-09-06T20:15:54,498 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:15:54,499 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5d57636bfe1d4f159972e307b570ed3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5d57636bfe1d4f159972e307b570ed3e 2023-09-06T20:15:54,501 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/67b1e7fc7b334f8db5a99d50583ce0a0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/67b1e7fc7b334f8db5a99d50583ce0a0 2023-09-06T20:15:54,501 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/b14a23c51e2e4001bfde2f5e416fbd3e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/b14a23c51e2e4001bfde2f5e416fbd3e 2023-09-06T20:15:54,502 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a12abad821ad4e4cbc7173181963d6ce to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/a12abad821ad4e4cbc7173181963d6ce 2023-09-06T20:15:54,503 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/188b53f56434403b9c7c7a0406b203e8 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/188b53f56434403b9c7c7a0406b203e8 2023-09-06T20:15:54,504 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/ca9024745bd5403b8710cd6eaac37a9a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/ca9024745bd5403b8710cd6eaac37a9a 2023-09-06T20:15:54,505 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/bdd4f68767094283b1e47be11f4576b9 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/bdd4f68767094283b1e47be11f4576b9 2023-09-06T20:15:54,506 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/73451464d4c14568b3a400f77974f09a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/73451464d4c14568b3a400f77974f09a 2023-09-06T20:15:54,507 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5dfa56afa1b94a63a781e5f87498788c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5dfa56afa1b94a63a781e5f87498788c 2023-09-06T20:15:54,507 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/48741bacb05b488c8788c928c6e40ebb to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/48741bacb05b488c8788c928c6e40ebb 2023-09-06T20:15:54,508 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5f38c0005a744187a593f2a6156d25de to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5f38c0005a744187a593f2a6156d25de 2023-09-06T20:15:54,509 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/025da8a1e54c46c8b56070f645e7f65f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/025da8a1e54c46c8b56070f645e7f65f 2023-09-06T20:15:54,510 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/538af06c9b064bb4bb9ad4cc245accf3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/538af06c9b064bb4bb9ad4cc245accf3 2023-09-06T20:15:54,511 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/4361d0a71493482da9192982363684e7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/4361d0a71493482da9192982363684e7 2023-09-06T20:15:54,512 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5f4d810d23a942f6bd86ad800cfdcdf4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5f4d810d23a942f6bd86ad800cfdcdf4 2023-09-06T20:15:54,513 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/f5faf9927d36420885601720ddbe7d58 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/f5faf9927d36420885601720ddbe7d58 2023-09-06T20:15:54,514 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/c09b4b4418ef47169f08152710065395 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/c09b4b4418ef47169f08152710065395 2023-09-06T20:15:54,515 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/3b5952b30d2d49088f5620f62a96b736 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/3b5952b30d2d49088f5620f62a96b736 2023-09-06T20:15:54,516 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/cb3bed0cc14c4b78a4d05d7f20eac783 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/cb3bed0cc14c4b78a4d05d7f20eac783 2023-09-06T20:15:54,517 DEBUG [StoreCloser-TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/fbf7a6d07e334a41a192dd8c5212c79a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/fbf7a6d07e334a41a192dd8c5212c79a 2023-09-06T20:15:54,520 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/recovered.edits/542.seqid, newMaxSeqId=542, maxSeqId=1 2023-09-06T20:15:54,521 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b. 2023-09-06T20:15:54,521 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1622): Region close journal for 969b36246b1411b7cf0e9c1debb5278b: 2023-09-06T20:15:54,523 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] handler.UnassignRegionHandler(163): Closed 969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:54,523 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=163 updating hbase:meta row=969b36246b1411b7cf0e9c1debb5278b, regionState=CLOSED 2023-09-06T20:15:54,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=164, resume processing ppid=163 2023-09-06T20:15:54,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=164, ppid=163, state=SUCCESS; CloseRegionProcedure 969b36246b1411b7cf0e9c1debb5278b, server=jenkins-hbase3.apache.org,33275,1694031217268 in 1.2010 sec 2023-09-06T20:15:54,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=163, resume processing ppid=162 2023-09-06T20:15:54,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=163, ppid=162, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=969b36246b1411b7cf0e9c1debb5278b, UNASSIGN in 1.2050 sec 2023-09-06T20:15:54,527 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031354527"}]},"ts":"1694031354527"} 2023-09-06T20:15:54,528 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2023-09-06T20:15:54,529 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2023-09-06T20:15:54,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=162, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.2170 sec 2023-09-06T20:15:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=162 2023-09-06T20:15:55,421 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 162 completed 2023-09-06T20:15:55,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete TestAcidGuarantees 2023-09-06T20:15:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=165, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:55,423 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=165, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=165 2023-09-06T20:15:55,424 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=165, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:55,427 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:55,429 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/recovered.edits] 2023-09-06T20:15:55,432 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/1b584078b1614222898e77d305d523da to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/1b584078b1614222898e77d305d523da 2023-09-06T20:15:55,433 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bf813ee285b344de88682044b06f2975 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/A/bf813ee285b344de88682044b06f2975 2023-09-06T20:15:55,435 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/1e1a265e22e54b8ba9ed7aafdb01b370 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/1e1a265e22e54b8ba9ed7aafdb01b370 2023-09-06T20:15:55,436 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/436553133a5d454fa34ee2b022df017b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/B/436553133a5d454fa34ee2b022df017b 2023-09-06T20:15:55,438 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/07082a34048b4744892478f12e2c2973 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/07082a34048b4744892478f12e2c2973 2023-09-06T20:15:55,439 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/22e25336e02f49168930c0a8d94b3e88 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/22e25336e02f49168930c0a8d94b3e88 2023-09-06T20:15:55,441 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5beb593455074516b7de0c081dfa6b68 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5beb593455074516b7de0c081dfa6b68 2023-09-06T20:15:55,442 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5c2512d6ddc54291b1abdc9d80d2041e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/5c2512d6ddc54291b1abdc9d80d2041e 2023-09-06T20:15:55,443 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/612ec5136433423b87b8b9ae20a70295 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/612ec5136433423b87b8b9ae20a70295 2023-09-06T20:15:55,443 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/f7a4dab53c964d42ad58baf90b2da106 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/C/f7a4dab53c964d42ad58baf90b2da106 2023-09-06T20:15:55,446 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/recovered.edits/542.seqid to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b/recovered.edits/542.seqid 2023-09-06T20:15:55,446 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/969b36246b1411b7cf0e9c1debb5278b 2023-09-06T20:15:55,446 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2023-09-06T20:15:55,448 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=165, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:55,449 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2023-09-06T20:15:55,451 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2023-09-06T20:15:55,451 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=165, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:55,452 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2023-09-06T20:15:55,452 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694031355452"}]},"ts":"9223372036854775807"} 2023-09-06T20:15:55,453 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:15:55,453 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 969b36246b1411b7cf0e9c1debb5278b, NAME => 'TestAcidGuarantees,,1694031330612.969b36246b1411b7cf0e9c1debb5278b.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:15:55,453 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2023-09-06T20:15:55,453 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694031355453"}]},"ts":"9223372036854775807"} 2023-09-06T20:15:55,454 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2023-09-06T20:15:55,456 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=165, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:55,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=165, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 35 msec 2023-09-06T20:15:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=165 2023-09-06T20:15:55,525 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 165 completed 2023-09-06T20:15:55,537 INFO [Listener at localhost/35961 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testScanAtomicity Thread=222 (was 222), OpenFileDescriptor=625 (was 568) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=560 (was 609), ProcessCount=167 (was 167), AvailableMemoryMB=4861 (was 4877) 2023-09-06T20:15:55,549 INFO [Listener at localhost/35961 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testMobGetAtomicity Thread=222, OpenFileDescriptor=625, MaxFileDescriptor=60000, SystemLoadAverage=560, ProcessCount=167, AvailableMemoryMB=4861 2023-09-06T20:15:55,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-06T20:15:55,552 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:15:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=166, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:55,554 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=166, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:15:55,554 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:55,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 166 2023-09-06T20:15:55,554 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=166, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:15:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=166 2023-09-06T20:15:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=166 2023-09-06T20:15:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=166 2023-09-06T20:15:55,961 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236 2023-09-06T20:15:55,965 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:15:55,965 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1668): Closing 7062ed4ddcd76aee2adae17e574b6875, disabling compactions & flushes 2023-09-06T20:15:55,965 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:55,965 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:55,965 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. after waiting 0 ms 2023-09-06T20:15:55,965 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:55,965 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:55,965 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:15:55,966 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=166, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:15:55,967 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1694031355966"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694031355966"}]},"ts":"1694031355966"} 2023-09-06T20:15:55,968 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:15:55,968 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=166, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:15:55,968 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031355968"}]},"ts":"1694031355968"} 2023-09-06T20:15:55,969 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2023-09-06T20:15:55,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=7062ed4ddcd76aee2adae17e574b6875, ASSIGN}] 2023-09-06T20:15:55,975 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=167, ppid=166, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=7062ed4ddcd76aee2adae17e574b6875, ASSIGN 2023-09-06T20:15:55,975 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=167, ppid=166, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7062ed4ddcd76aee2adae17e574b6875, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33275,1694031217268; forceNewPlan=false, retain=false 2023-09-06T20:15:56,127 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=167 updating hbase:meta row=7062ed4ddcd76aee2adae17e574b6875, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:56,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE; OpenRegionProcedure 7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:15:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=166 2023-09-06T20:15:56,282 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:56,282 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(7254): Opening region: {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:15:56,282 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:56,282 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:15:56,282 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(7296): checking encryption for 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:56,282 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(7299): checking classloading for 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:56,283 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:56,284 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:56,284 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7062ed4ddcd76aee2adae17e574b6875 columnFamilyName A 2023-09-06T20:15:56,285 DEBUG [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:56,285 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.HStore(324): Store=7062ed4ddcd76aee2adae17e574b6875/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:56,285 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:56,286 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:56,286 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7062ed4ddcd76aee2adae17e574b6875 columnFamilyName B 2023-09-06T20:15:56,287 DEBUG [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:56,287 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.HStore(324): Store=7062ed4ddcd76aee2adae17e574b6875/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:56,287 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:56,288 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:56,288 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7062ed4ddcd76aee2adae17e574b6875 columnFamilyName C 2023-09-06T20:15:56,288 DEBUG [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:56,288 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.HStore(324): Store=7062ed4ddcd76aee2adae17e574b6875/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:56,289 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:56,289 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:56,289 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:56,290 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-06T20:15:56,291 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(1076): writing seq id for 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:56,293 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:15:56,293 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(1093): Opened 7062ed4ddcd76aee2adae17e574b6875; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64466992, jitterRate=-0.039366960525512695}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-06T20:15:56,294 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(992): Region open journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:15:56,295 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., pid=168, masterSystemTime=1694031356279 2023-09-06T20:15:56,296 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:56,296 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:56,296 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=167 updating hbase:meta row=7062ed4ddcd76aee2adae17e574b6875, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:56,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=168, resume processing ppid=167 2023-09-06T20:15:56,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=168, ppid=167, state=SUCCESS; OpenRegionProcedure 7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 in 169 msec 2023-09-06T20:15:56,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=167, resume processing ppid=166 2023-09-06T20:15:56,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=167, ppid=166, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=7062ed4ddcd76aee2adae17e574b6875, ASSIGN in 326 msec 2023-09-06T20:15:56,300 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=166, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:15:56,301 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031356300"}]},"ts":"1694031356300"} 2023-09-06T20:15:56,301 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2023-09-06T20:15:56,304 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=166, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:15:56,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=166, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 752 msec 2023-09-06T20:15:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=166 2023-09-06T20:15:56,658 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 166 completed 2023-09-06T20:15:56,659 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x18fcc509 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:56,664 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f044e50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:56,666 DEBUG [Listener at localhost/35961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:56,667 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45564, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:56,668 DEBUG [Listener at localhost/35961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:15:56,669 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51246, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:15:56,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-06T20:15:56,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$13(2700): Client=jenkins//172.31.12.81 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:15:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=169, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2023-09-06T20:15:57,082 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.963 2023-09-06T20:15:57,082 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.999 2023-09-06T20:15:57,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2023-09-06T20:15:57,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=7062ed4ddcd76aee2adae17e574b6875, REOPEN/MOVE}] 2023-09-06T20:15:57,087 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=171, ppid=170, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=7062ed4ddcd76aee2adae17e574b6875, REOPEN/MOVE 2023-09-06T20:15:57,087 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=171 updating hbase:meta row=7062ed4ddcd76aee2adae17e574b6875, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:57,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE; CloseRegionProcedure 7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:15:57,243 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] handler.UnassignRegionHandler(119): Close 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:57,244 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegion(1668): Closing 7062ed4ddcd76aee2adae17e574b6875, disabling compactions & flushes 2023-09-06T20:15:57,244 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:57,244 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:57,244 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. after waiting 0 ms 2023-09-06T20:15:57,244 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:57,247 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:15:57,248 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:57,248 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegion(1622): Region close journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:15:57,248 WARN [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegionServer(3735): Not adding moved region record: 7062ed4ddcd76aee2adae17e574b6875 to self. 2023-09-06T20:15:57,250 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] handler.UnassignRegionHandler(163): Closed 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:57,250 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=171 updating hbase:meta row=7062ed4ddcd76aee2adae17e574b6875, regionState=CLOSED 2023-09-06T20:15:57,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=172, resume processing ppid=171 2023-09-06T20:15:57,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=172, ppid=171, state=SUCCESS; CloseRegionProcedure 7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 in 163 msec 2023-09-06T20:15:57,253 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=171, ppid=170, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7062ed4ddcd76aee2adae17e574b6875, REOPEN/MOVE; state=CLOSED, location=jenkins-hbase3.apache.org,33275,1694031217268; forceNewPlan=false, retain=true 2023-09-06T20:15:57,403 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=171 updating hbase:meta row=7062ed4ddcd76aee2adae17e574b6875, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:57,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=173, ppid=171, state=RUNNABLE; OpenRegionProcedure 7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:15:57,558 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:57,558 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(7254): Opening region: {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:15:57,559 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:57,559 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:15:57,559 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(7296): checking encryption for 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:57,559 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(7299): checking classloading for 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:57,560 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:57,561 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:57,561 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7062ed4ddcd76aee2adae17e574b6875 columnFamilyName A 2023-09-06T20:15:57,562 DEBUG [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:57,562 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.HStore(324): Store=7062ed4ddcd76aee2adae17e574b6875/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:57,562 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:57,563 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:57,563 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7062ed4ddcd76aee2adae17e574b6875 columnFamilyName B 2023-09-06T20:15:57,563 DEBUG [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:57,563 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.HStore(324): Store=7062ed4ddcd76aee2adae17e574b6875/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:57,563 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:57,564 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-06T20:15:57,564 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7062ed4ddcd76aee2adae17e574b6875 columnFamilyName C 2023-09-06T20:15:57,564 DEBUG [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:15:57,564 INFO [StoreOpener-7062ed4ddcd76aee2adae17e574b6875-1 {}] regionserver.HStore(324): Store=7062ed4ddcd76aee2adae17e574b6875/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:15:57,564 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:57,565 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:57,566 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:57,567 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-06T20:15:57,569 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(1076): writing seq id for 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:57,570 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(1093): Opened 7062ed4ddcd76aee2adae17e574b6875; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70644633, jitterRate=0.052687063813209534}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-06T20:15:57,570 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(992): Region open journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:15:57,571 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., pid=173, masterSystemTime=1694031357556 2023-09-06T20:15:57,572 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:57,572 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:57,572 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=171 updating hbase:meta row=7062ed4ddcd76aee2adae17e574b6875, regionState=OPEN, openSeqNum=5, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:57,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=173, resume processing ppid=171 2023-09-06T20:15:57,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=173, ppid=171, state=SUCCESS; OpenRegionProcedure 7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 in 169 msec 2023-09-06T20:15:57,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=171, resume processing ppid=170 2023-09-06T20:15:57,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=171, ppid=170, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=7062ed4ddcd76aee2adae17e574b6875, REOPEN/MOVE in 488 msec 2023-09-06T20:15:57,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=170, resume processing ppid=169 2023-09-06T20:15:57,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=170, ppid=169, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2023-09-06T20:15:57,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=169, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2023-09-06T20:15:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=169 2023-09-06T20:15:57,580 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x28fecf47 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:57,584 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@565483dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:57,585 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6090edd2 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:57,593 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4a8270, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:57,593 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0ef38761 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:57,597 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@696a0510, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:57,598 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5a125dbe to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:57,601 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48480112, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:57,602 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3839a9cc to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:57,605 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708efcd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:57,606 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x62e4708a to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:57,613 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658b2832, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:57,614 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0d3acf9e to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:57,619 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c1b9e93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:57,619 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2876f366 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:57,628 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736b8fc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:57,629 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7c4a00fc to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:57,632 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@633831d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:57,633 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0c17d3e8 to 127.0.0.1:58481 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:15:57,638 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20f66f76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-06T20:15:57,651 DEBUG [hconnection-0x764fccb2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:57,651 DEBUG [hconnection-0x2350cd93-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:57,653 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45572, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:57,653 DEBUG [hconnection-0x7dfddae2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:57,654 DEBUG [hconnection-0x109e9b69-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:57,654 DEBUG [hconnection-0x616fb68c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:57,654 DEBUG [hconnection-0x4115c14a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:57,655 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45602, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:57,657 DEBUG [hconnection-0x4fd14414-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:57,658 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45588, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:57,658 DEBUG [hconnection-0x2a943f22-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:57,659 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45614, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:57,659 DEBUG [hconnection-0x12ea2c10-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:57,661 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45568, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:57,661 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45596, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:57,662 DEBUG [hconnection-0x776d4ce8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:15:57,662 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45630, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:57,662 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45642, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:57,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:57,670 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45652, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:57,670 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45664, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:15:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2023-09-06T20:15:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=174 2023-09-06T20:15:57,674 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:57,674 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:57,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:57,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2023-09-06T20:15:57,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:15:57,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:57,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:15:57,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:57,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:15:57,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:57,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031417693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:57,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031417693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:57,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:57,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031417695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031417694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:57,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031417695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:57,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090650dc2bd3796c449eabade885147703f0_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031357679/Put/seqid=0 2023-09-06T20:15:57,714 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090650dc2bd3796c449eabade885147703f0_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090650dc2bd3796c449eabade885147703f0_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:57,715 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/a2f9cf3c38cf437fb2cb11622f09b132, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:15:57,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/a2f9cf3c38cf437fb2cb11622f09b132 is 175, key is test_row_0/A:col10/1694031357679/Put/seqid=0 2023-09-06T20:15:57,731 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=21, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/a2f9cf3c38cf437fb2cb11622f09b132 2023-09-06T20:15:57,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/d54d79254cc14740ad84358d6cae82f6 is 50, key is test_row_0/B:col10/1694031357679/Put/seqid=0 2023-09-06T20:15:57,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/d54d79254cc14740ad84358d6cae82f6 2023-09-06T20:15:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=174 2023-09-06T20:15:57,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/568697960a3b489698903a899f3f2b80 is 50, key is test_row_0/C:col10/1694031357679/Put/seqid=0 2023-09-06T20:15:57,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/568697960a3b489698903a899f3f2b80 2023-09-06T20:15:57,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031417797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:57,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031417798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:57,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/a2f9cf3c38cf437fb2cb11622f09b132 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/a2f9cf3c38cf437fb2cb11622f09b132 2023-09-06T20:15:57,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031417798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:57,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031417799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:57,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/a2f9cf3c38cf437fb2cb11622f09b132, entries=100, sequenceid=21, filesize=21.8 K 2023-09-06T20:15:57,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/d54d79254cc14740ad84358d6cae82f6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d54d79254cc14740ad84358d6cae82f6 2023-09-06T20:15:57,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031417805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:57,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d54d79254cc14740ad84358d6cae82f6, entries=100, sequenceid=21, filesize=9.4 K 2023-09-06T20:15:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/568697960a3b489698903a899f3f2b80 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/568697960a3b489698903a899f3f2b80 2023-09-06T20:15:57,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/568697960a3b489698903a899f3f2b80, entries=100, sequenceid=21, filesize=9.4 K 2023-09-06T20:15:57,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 7062ed4ddcd76aee2adae17e574b6875 in 134ms, sequenceid=21, compaction requested=false 2023-09-06T20:15:57,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:15:57,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2023-09-06T20:15:57,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:57,827 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2023-09-06T20:15:57,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:15:57,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:57,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:15:57,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:57,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:15:57,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:57,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309062f661341dccd43c8b73c23e6be2ad384_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031357693/Put/seqid=0 2023-09-06T20:15:57,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=174 2023-09-06T20:15:58,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:15:58,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031418033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031418033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031418035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031418037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031418037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031418138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031418138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031418138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031418139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031418140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,249 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309062f661341dccd43c8b73c23e6be2ad384_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309062f661341dccd43c8b73c23e6be2ad384_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:58,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/11ff455875ef41da86eef1d6b58fb0ba, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:15:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/11ff455875ef41da86eef1d6b58fb0ba is 175, key is test_row_0/A:col10/1694031357693/Put/seqid=0 2023-09-06T20:15:58,256 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/11ff455875ef41da86eef1d6b58fb0ba 2023-09-06T20:15:58,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/b09c5c4349e84c03a031f8be3f13b3f2 is 50, key is test_row_0/B:col10/1694031357693/Put/seqid=0 2023-09-06T20:15:58,271 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/b09c5c4349e84c03a031f8be3f13b3f2 2023-09-06T20:15:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=174 2023-09-06T20:15:58,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/ec11dc905e6f4aceac00423e3ea857ea is 50, key is test_row_0/C:col10/1694031357693/Put/seqid=0 2023-09-06T20:15:58,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031418340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031418340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031418341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031418342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031418343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031418643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031418644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031418644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031418645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031418647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:58,682 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/ec11dc905e6f4aceac00423e3ea857ea 2023-09-06T20:15:58,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/11ff455875ef41da86eef1d6b58fb0ba as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/11ff455875ef41da86eef1d6b58fb0ba 2023-09-06T20:15:58,690 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/11ff455875ef41da86eef1d6b58fb0ba, entries=150, sequenceid=41, filesize=30.2 K 2023-09-06T20:15:58,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/b09c5c4349e84c03a031f8be3f13b3f2 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b09c5c4349e84c03a031f8be3f13b3f2 2023-09-06T20:15:58,695 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b09c5c4349e84c03a031f8be3f13b3f2, entries=150, sequenceid=41, filesize=11.7 K 2023-09-06T20:15:58,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/ec11dc905e6f4aceac00423e3ea857ea as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ec11dc905e6f4aceac00423e3ea857ea 2023-09-06T20:15:58,699 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ec11dc905e6f4aceac00423e3ea857ea, entries=150, sequenceid=41, filesize=11.7 K 2023-09-06T20:15:58,700 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3022): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 7062ed4ddcd76aee2adae17e574b6875 in 872ms, sequenceid=41, compaction requested=false 2023-09-06T20:15:58,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:15:58,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:58,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=175 2023-09-06T20:15:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=175 2023-09-06T20:15:58,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=175, resume processing ppid=174 2023-09-06T20:15:58,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=175, ppid=174, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0270 sec 2023-09-06T20:15:58,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=174, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 1.0400 sec 2023-09-06T20:15:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=174 2023-09-06T20:15:58,776 INFO [Thread-3234 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 174 completed 2023-09-06T20:15:58,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2023-09-06T20:15:58,781 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=176 2023-09-06T20:15:58,781 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:58,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=176 2023-09-06T20:15:58,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2023-09-06T20:15:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:58,934 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2023-09-06T20:15:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:15:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:15:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:15:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:58,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309064f85cce75115473587d6cf4343fe1123_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031358035/Put/seqid=0 2023-09-06T20:15:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=176 2023-09-06T20:15:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:59,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:15:59,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031419160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031419162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031419162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031419163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031419163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031419264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031419267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031419267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031419267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031419267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,355 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309064f85cce75115473587d6cf4343fe1123_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309064f85cce75115473587d6cf4343fe1123_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:59,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/afd809d30e294794961b123369f6cded, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:15:59,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/afd809d30e294794961b123369f6cded is 175, key is test_row_0/A:col10/1694031358035/Put/seqid=0 2023-09-06T20:15:59,363 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=58, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/afd809d30e294794961b123369f6cded 2023-09-06T20:15:59,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/7bb3afd7b4944d5fb4a3119e72133a03 is 50, key is test_row_0/B:col10/1694031358035/Put/seqid=0 2023-09-06T20:15:59,376 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/7bb3afd7b4944d5fb4a3119e72133a03 2023-09-06T20:15:59,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/0ffb93c767864762a6530392cade6eac is 50, key is test_row_0/C:col10/1694031358035/Put/seqid=0 2023-09-06T20:15:59,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=176 2023-09-06T20:15:59,387 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/0ffb93c767864762a6530392cade6eac 2023-09-06T20:15:59,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/afd809d30e294794961b123369f6cded as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/afd809d30e294794961b123369f6cded 2023-09-06T20:15:59,395 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/afd809d30e294794961b123369f6cded, entries=150, sequenceid=58, filesize=30.2 K 2023-09-06T20:15:59,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/7bb3afd7b4944d5fb4a3119e72133a03 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7bb3afd7b4944d5fb4a3119e72133a03 2023-09-06T20:15:59,399 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7bb3afd7b4944d5fb4a3119e72133a03, entries=150, sequenceid=58, filesize=11.7 K 2023-09-06T20:15:59,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/0ffb93c767864762a6530392cade6eac as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/0ffb93c767864762a6530392cade6eac 2023-09-06T20:15:59,403 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/0ffb93c767864762a6530392cade6eac, entries=150, sequenceid=58, filesize=11.7 K 2023-09-06T20:15:59,404 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3022): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 7062ed4ddcd76aee2adae17e574b6875 in 469ms, sequenceid=58, compaction requested=true 2023-09-06T20:15:59,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:15:59,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:15:59,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=177 2023-09-06T20:15:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=177 2023-09-06T20:15:59,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=177, resume processing ppid=176 2023-09-06T20:15:59,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=177, ppid=176, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 624 msec 2023-09-06T20:15:59,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=176, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 629 msec 2023-09-06T20:15:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:59,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2023-09-06T20:15:59,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:15:59,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:59,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:15:59,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:59,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:15:59,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:15:59,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309060d808c7eae214372b72bd067d23ce453_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031359467/Put/seqid=0 2023-09-06T20:15:59,483 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309060d808c7eae214372b72bd067d23ce453_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309060d808c7eae214372b72bd067d23ce453_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:15:59,484 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/ed6d6a8bbca74397b5a741d1b9d48a01, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:15:59,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/ed6d6a8bbca74397b5a741d1b9d48a01 is 175, key is test_row_0/A:col10/1694031359467/Put/seqid=0 2023-09-06T20:15:59,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031419483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031419483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031419484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031419485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031419485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031419597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031419597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031419597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031419598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031419598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031419800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031419801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031419801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031419801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:15:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031419802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:15:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=176 2023-09-06T20:15:59,884 INFO [Thread-3234 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 176 completed 2023-09-06T20:15:59,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:15:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees 2023-09-06T20:15:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=178 2023-09-06T20:15:59,887 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:15:59,888 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=178, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:15:59,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:15:59,890 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/ed6d6a8bbca74397b5a741d1b9d48a01 2023-09-06T20:15:59,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/270695a6ac344b318ea54e7b4e90dad9 is 50, key is test_row_0/B:col10/1694031359467/Put/seqid=0 2023-09-06T20:15:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=178 2023-09-06T20:16:00,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2023-09-06T20:16:00,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:00,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:00,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:00,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(53): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:00,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:00,050 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-06T20:16:00,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031420104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031420104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031420105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031420106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031420107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=178 2023-09-06T20:16:00,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2023-09-06T20:16:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:00,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(53): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:00,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:00,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/270695a6ac344b318ea54e7b4e90dad9 2023-09-06T20:16:00,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/6ba02f1841014882a53e8b3f9ec29ced is 50, key is test_row_0/C:col10/1694031359467/Put/seqid=0 2023-09-06T20:16:00,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/6ba02f1841014882a53e8b3f9ec29ced 2023-09-06T20:16:00,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/ed6d6a8bbca74397b5a741d1b9d48a01 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/ed6d6a8bbca74397b5a741d1b9d48a01 2023-09-06T20:16:00,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/ed6d6a8bbca74397b5a741d1b9d48a01, entries=150, sequenceid=79, filesize=30.2 K 2023-09-06T20:16:00,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/270695a6ac344b318ea54e7b4e90dad9 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/270695a6ac344b318ea54e7b4e90dad9 2023-09-06T20:16:00,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/270695a6ac344b318ea54e7b4e90dad9, entries=150, sequenceid=79, filesize=11.7 K 2023-09-06T20:16:00,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/6ba02f1841014882a53e8b3f9ec29ced as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/6ba02f1841014882a53e8b3f9ec29ced 2023-09-06T20:16:00,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/6ba02f1841014882a53e8b3f9ec29ced, entries=150, sequenceid=79, filesize=11.7 K 2023-09-06T20:16:00,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 7062ed4ddcd76aee2adae17e574b6875 in 863ms, sequenceid=79, compaction requested=true 2023-09-06T20:16:00,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:00,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:16:00,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:00,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:16:00,331 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:16:00,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:00,331 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:16:00,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:16:00,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:00,333 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 115158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:16:00,333 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:16:00,333 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/A is initiating minor compaction (all files) 2023-09-06T20:16:00,333 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/B is initiating minor compaction (all files) 2023-09-06T20:16:00,333 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/A in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:00,333 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/B in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:00,333 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/a2f9cf3c38cf437fb2cb11622f09b132, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/11ff455875ef41da86eef1d6b58fb0ba, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/afd809d30e294794961b123369f6cded, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/ed6d6a8bbca74397b5a741d1b9d48a01] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=112.5 K 2023-09-06T20:16:00,333 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d54d79254cc14740ad84358d6cae82f6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b09c5c4349e84c03a031f8be3f13b3f2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7bb3afd7b4944d5fb4a3119e72133a03, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/270695a6ac344b318ea54e7b4e90dad9] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=44.5 K 2023-09-06T20:16:00,333 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:00,333 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/a2f9cf3c38cf437fb2cb11622f09b132, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/11ff455875ef41da86eef1d6b58fb0ba, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/afd809d30e294794961b123369f6cded, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/ed6d6a8bbca74397b5a741d1b9d48a01] 2023-09-06T20:16:00,334 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting d54d79254cc14740ad84358d6cae82f6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1694031357679 2023-09-06T20:16:00,334 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting a2f9cf3c38cf437fb2cb11622f09b132, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1694031357679 2023-09-06T20:16:00,334 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 11ff455875ef41da86eef1d6b58fb0ba, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1694031357690 2023-09-06T20:16:00,334 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b09c5c4349e84c03a031f8be3f13b3f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1694031357690 2023-09-06T20:16:00,335 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting afd809d30e294794961b123369f6cded, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1694031358031 2023-09-06T20:16:00,335 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 7bb3afd7b4944d5fb4a3119e72133a03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1694031358031 2023-09-06T20:16:00,335 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting ed6d6a8bbca74397b5a741d1b9d48a01, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1694031359161 2023-09-06T20:16:00,335 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 270695a6ac344b318ea54e7b4e90dad9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1694031359161 2023-09-06T20:16:00,346 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:00,347 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#B#compaction#546 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:00,348 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/9618e7d10f96414087781b95a8bb6ca3 is 50, key is test_row_0/B:col10/1694031359467/Put/seqid=0 2023-09-06T20:16:00,350 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309066a1b888d787f48c0889ee75e510916f7_7062ed4ddcd76aee2adae17e574b6875 store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:00,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2023-09-06T20:16:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:00,351 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2023-09-06T20:16:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:00,352 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309066a1b888d787f48c0889ee75e510916f7_7062ed4ddcd76aee2adae17e574b6875, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:00,352 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309066a1b888d787f48c0889ee75e510916f7_7062ed4ddcd76aee2adae17e574b6875 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:00,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090680c580466de04bb4b6b2652dea32eb80_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031359477/Put/seqid=0 2023-09-06T20:16:00,370 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/9618e7d10f96414087781b95a8bb6ca3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9618e7d10f96414087781b95a8bb6ca3 2023-09-06T20:16:00,374 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/B of 7062ed4ddcd76aee2adae17e574b6875 into 9618e7d10f96414087781b95a8bb6ca3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:00,374 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:00,374 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/B, priority=12, startTime=1694031360331; duration=0sec 2023-09-06T20:16:00,374 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:00,374 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:B 2023-09-06T20:16:00,374 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:16:00,376 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:16:00,376 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/C is initiating minor compaction (all files) 2023-09-06T20:16:00,376 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/C in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:00,376 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/568697960a3b489698903a899f3f2b80, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ec11dc905e6f4aceac00423e3ea857ea, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/0ffb93c767864762a6530392cade6eac, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/6ba02f1841014882a53e8b3f9ec29ced] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=44.5 K 2023-09-06T20:16:00,376 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 568697960a3b489698903a899f3f2b80, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1694031357679 2023-09-06T20:16:00,376 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting ec11dc905e6f4aceac00423e3ea857ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1694031357690 2023-09-06T20:16:00,377 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 0ffb93c767864762a6530392cade6eac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1694031358031 2023-09-06T20:16:00,377 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 6ba02f1841014882a53e8b3f9ec29ced, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1694031359161 2023-09-06T20:16:00,384 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#C#compaction#549 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:00,385 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/5207f13a82914e1588d45deca1413613 is 50, key is test_row_0/C:col10/1694031359467/Put/seqid=0 2023-09-06T20:16:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=178 2023-09-06T20:16:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:00,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:00,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031420623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031420626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031420626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031420627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031420627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031420728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031420730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031420730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031420731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031420731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,761 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#A#compaction#547 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:00,761 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/1d5cb59d446c4aee9ce847d505638c02 is 175, key is test_row_0/A:col10/1694031359467/Put/seqid=0 2023-09-06T20:16:00,771 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/1d5cb59d446c4aee9ce847d505638c02 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/1d5cb59d446c4aee9ce847d505638c02 2023-09-06T20:16:00,776 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090680c580466de04bb4b6b2652dea32eb80_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090680c580466de04bb4b6b2652dea32eb80_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:00,777 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/A of 7062ed4ddcd76aee2adae17e574b6875 into 1d5cb59d446c4aee9ce847d505638c02(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:00,777 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:00,777 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/A, priority=12, startTime=1694031360331; duration=0sec 2023-09-06T20:16:00,777 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:00,777 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:A 2023-09-06T20:16:00,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/da25c95ebb154347992dc98fef2fae2b, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:00,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/da25c95ebb154347992dc98fef2fae2b is 175, key is test_row_0/A:col10/1694031359477/Put/seqid=0 2023-09-06T20:16:00,792 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/5207f13a82914e1588d45deca1413613 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/5207f13a82914e1588d45deca1413613 2023-09-06T20:16:00,797 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/C of 7062ed4ddcd76aee2adae17e574b6875 into 5207f13a82914e1588d45deca1413613(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:00,797 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:00,797 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/C, priority=12, startTime=1694031360331; duration=0sec 2023-09-06T20:16:00,797 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:00,797 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:C 2023-09-06T20:16:00,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031420931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031420932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031420932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031420933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:00,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031420933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=178 2023-09-06T20:16:01,182 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/da25c95ebb154347992dc98fef2fae2b 2023-09-06T20:16:01,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/9fb120009cce4e418af659a1e01c84c0 is 50, key is test_row_0/B:col10/1694031359477/Put/seqid=0 2023-09-06T20:16:01,192 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/9fb120009cce4e418af659a1e01c84c0 2023-09-06T20:16:01,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/1fe21f662f144807b02132fbd280a14d is 50, key is test_row_0/C:col10/1694031359477/Put/seqid=0 2023-09-06T20:16:01,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031421235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031421236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031421236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031421236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031421236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,602 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/1fe21f662f144807b02132fbd280a14d 2023-09-06T20:16:01,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/da25c95ebb154347992dc98fef2fae2b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/da25c95ebb154347992dc98fef2fae2b 2023-09-06T20:16:01,610 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/da25c95ebb154347992dc98fef2fae2b, entries=150, sequenceid=95, filesize=30.2 K 2023-09-06T20:16:01,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/9fb120009cce4e418af659a1e01c84c0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9fb120009cce4e418af659a1e01c84c0 2023-09-06T20:16:01,615 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9fb120009cce4e418af659a1e01c84c0, entries=150, sequenceid=95, filesize=11.7 K 2023-09-06T20:16:01,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/1fe21f662f144807b02132fbd280a14d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/1fe21f662f144807b02132fbd280a14d 2023-09-06T20:16:01,619 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/1fe21f662f144807b02132fbd280a14d, entries=150, sequenceid=95, filesize=11.7 K 2023-09-06T20:16:01,620 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 7062ed4ddcd76aee2adae17e574b6875 in 1269ms, sequenceid=95, compaction requested=false 2023-09-06T20:16:01,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:01,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:01,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=179 2023-09-06T20:16:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=179 2023-09-06T20:16:01,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=179, resume processing ppid=178 2023-09-06T20:16:01,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=179, ppid=178, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7330 sec 2023-09-06T20:16:01,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=178, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees in 1.7380 sec 2023-09-06T20:16:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:01,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2023-09-06T20:16:01,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:01,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:01,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:01,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:01,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:01,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:01,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906fa691d42d5504784a1709c2a49f154ac_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031361739/Put/seqid=0 2023-09-06T20:16:01,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031421748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031421749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031421749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,754 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906fa691d42d5504784a1709c2a49f154ac_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906fa691d42d5504784a1709c2a49f154ac_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031421751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031421751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,754 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/24cef1903e0c4ede91f9533e0fe940a1, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:01,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/24cef1903e0c4ede91f9533e0fe940a1 is 175, key is test_row_0/A:col10/1694031361739/Put/seqid=0 2023-09-06T20:16:01,763 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/24cef1903e0c4ede91f9533e0fe940a1 2023-09-06T20:16:01,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/e7197645ad1c4b7d9a72212a823ee0a0 is 50, key is test_row_0/B:col10/1694031361739/Put/seqid=0 2023-09-06T20:16:01,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031421852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031421853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031421854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031421855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031421855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=178 2023-09-06T20:16:01,992 INFO [Thread-3234 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 178 completed 2023-09-06T20:16:01,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:16:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees 2023-09-06T20:16:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=180 2023-09-06T20:16:01,996 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:16:01,996 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=180, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:16:01,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:16:02,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031422056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031422056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031422057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031422058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031422059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=180 2023-09-06T20:16:02,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2023-09-06T20:16:02,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:02,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:02,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:02,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(53): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:02,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:02,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/e7197645ad1c4b7d9a72212a823ee0a0 2023-09-06T20:16:02,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/176bf07c580f41ca9b3b2b75af92a4d4 is 50, key is test_row_0/C:col10/1694031361739/Put/seqid=0 2023-09-06T20:16:02,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/176bf07c580f41ca9b3b2b75af92a4d4 2023-09-06T20:16:02,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/24cef1903e0c4ede91f9533e0fe940a1 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/24cef1903e0c4ede91f9533e0fe940a1 2023-09-06T20:16:02,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/24cef1903e0c4ede91f9533e0fe940a1, entries=150, sequenceid=120, filesize=30.2 K 2023-09-06T20:16:02,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/e7197645ad1c4b7d9a72212a823ee0a0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/e7197645ad1c4b7d9a72212a823ee0a0 2023-09-06T20:16:02,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/e7197645ad1c4b7d9a72212a823ee0a0, entries=150, sequenceid=120, filesize=11.7 K 2023-09-06T20:16:02,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/176bf07c580f41ca9b3b2b75af92a4d4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/176bf07c580f41ca9b3b2b75af92a4d4 2023-09-06T20:16:02,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/176bf07c580f41ca9b3b2b75af92a4d4, entries=150, sequenceid=120, filesize=11.7 K 2023-09-06T20:16:02,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 7062ed4ddcd76aee2adae17e574b6875 in 468ms, sequenceid=120, compaction requested=true 2023-09-06T20:16:02,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:02,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:16:02,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:02,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:16:02,208 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:16:02,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:02,208 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:16:02,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:16:02,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:02,209 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:16:02,209 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/A is initiating minor compaction (all files) 2023-09-06T20:16:02,209 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36090 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:16:02,209 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/A in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:02,209 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/B is initiating minor compaction (all files) 2023-09-06T20:16:02,209 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/B in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:02,209 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/1d5cb59d446c4aee9ce847d505638c02, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/da25c95ebb154347992dc98fef2fae2b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/24cef1903e0c4ede91f9533e0fe940a1] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=90.8 K 2023-09-06T20:16:02,209 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:02,209 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9618e7d10f96414087781b95a8bb6ca3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9fb120009cce4e418af659a1e01c84c0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/e7197645ad1c4b7d9a72212a823ee0a0] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=35.2 K 2023-09-06T20:16:02,209 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/1d5cb59d446c4aee9ce847d505638c02, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/da25c95ebb154347992dc98fef2fae2b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/24cef1903e0c4ede91f9533e0fe940a1] 2023-09-06T20:16:02,210 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 9618e7d10f96414087781b95a8bb6ca3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1694031359161 2023-09-06T20:16:02,210 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1d5cb59d446c4aee9ce847d505638c02, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1694031359161 2023-09-06T20:16:02,210 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 9fb120009cce4e418af659a1e01c84c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1694031359477 2023-09-06T20:16:02,210 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting da25c95ebb154347992dc98fef2fae2b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1694031359477 2023-09-06T20:16:02,210 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting e7197645ad1c4b7d9a72212a823ee0a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1694031360622 2023-09-06T20:16:02,210 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 24cef1903e0c4ede91f9533e0fe940a1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1694031360622 2023-09-06T20:16:02,219 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:02,220 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#B#compaction#555 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:02,220 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/ba2a75fbc530463fa6fcc4d8dd629c4c is 50, key is test_row_0/B:col10/1694031361739/Put/seqid=0 2023-09-06T20:16:02,221 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2023090692e98bdd462543d4b30dce4fbfe7f2fd_7062ed4ddcd76aee2adae17e574b6875 store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:02,222 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2023090692e98bdd462543d4b30dce4fbfe7f2fd_7062ed4ddcd76aee2adae17e574b6875, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:02,223 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090692e98bdd462543d4b30dce4fbfe7f2fd_7062ed4ddcd76aee2adae17e574b6875 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:02,240 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#A#compaction#556 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:02,240 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/ebad05b8587642ec8d7e8450e02a0d7b is 175, key is test_row_0/A:col10/1694031361739/Put/seqid=0 2023-09-06T20:16:02,251 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/ebad05b8587642ec8d7e8450e02a0d7b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/ebad05b8587642ec8d7e8450e02a0d7b 2023-09-06T20:16:02,255 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/A of 7062ed4ddcd76aee2adae17e574b6875 into ebad05b8587642ec8d7e8450e02a0d7b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:02,255 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:02,255 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/A, priority=13, startTime=1694031362208; duration=0sec 2023-09-06T20:16:02,255 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:02,255 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:A 2023-09-06T20:16:02,255 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:16:02,255 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36090 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:16:02,255 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/C is initiating minor compaction (all files) 2023-09-06T20:16:02,256 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/C in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:02,256 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/5207f13a82914e1588d45deca1413613, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/1fe21f662f144807b02132fbd280a14d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/176bf07c580f41ca9b3b2b75af92a4d4] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=35.2 K 2023-09-06T20:16:02,256 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5207f13a82914e1588d45deca1413613, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1694031359161 2023-09-06T20:16:02,256 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1fe21f662f144807b02132fbd280a14d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1694031359477 2023-09-06T20:16:02,257 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 176bf07c580f41ca9b3b2b75af92a4d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1694031360622 2023-09-06T20:16:02,267 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#C#compaction#557 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:02,268 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/152d24a29c9e4b319cc6f8e6f3c413fa is 50, key is test_row_0/C:col10/1694031361739/Put/seqid=0 2023-09-06T20:16:02,289 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/152d24a29c9e4b319cc6f8e6f3c413fa as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/152d24a29c9e4b319cc6f8e6f3c413fa 2023-09-06T20:16:02,293 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/C of 7062ed4ddcd76aee2adae17e574b6875 into 152d24a29c9e4b319cc6f8e6f3c413fa(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:02,293 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:02,293 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/C, priority=13, startTime=1694031362208; duration=0sec 2023-09-06T20:16:02,293 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:02,293 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:C 2023-09-06T20:16:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=180 2023-09-06T20:16:02,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2023-09-06T20:16:02,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:02,302 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2023-09-06T20:16:02,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:02,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:02,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:02,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:02,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:02,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906482c6d971e2e43b1a6643c52c44d665b_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031361750/Put/seqid=0 2023-09-06T20:16:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:02,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:02,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031422429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031422432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031422433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031422433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031422433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031422534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031422537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031422537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031422538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031422538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=180 2023-09-06T20:16:02,638 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/ba2a75fbc530463fa6fcc4d8dd629c4c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/ba2a75fbc530463fa6fcc4d8dd629c4c 2023-09-06T20:16:02,642 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/B of 7062ed4ddcd76aee2adae17e574b6875 into ba2a75fbc530463fa6fcc4d8dd629c4c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:02,642 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:02,642 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/B, priority=13, startTime=1694031362208; duration=0sec 2023-09-06T20:16:02,642 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:02,643 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:B 2023-09-06T20:16:02,718 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906482c6d971e2e43b1a6643c52c44d665b_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906482c6d971e2e43b1a6643c52c44d665b_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:02,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/c6d639737f5343cf87f856a46827091c, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:02,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/c6d639737f5343cf87f856a46827091c is 175, key is test_row_0/A:col10/1694031361750/Put/seqid=0 2023-09-06T20:16:02,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031422737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031422740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031422740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031422740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:02,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031422741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031423041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031423043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031423044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031423044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031423046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=180 2023-09-06T20:16:03,136 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/c6d639737f5343cf87f856a46827091c 2023-09-06T20:16:03,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/09ffe86814b94b0f8523dbf9c2b14ff4 is 50, key is test_row_0/B:col10/1694031361750/Put/seqid=0 2023-09-06T20:16:03,148 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/09ffe86814b94b0f8523dbf9c2b14ff4 2023-09-06T20:16:03,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/9271d09d0bec454a8710c5fd815fa3b1 is 50, key is test_row_0/C:col10/1694031361750/Put/seqid=0 2023-09-06T20:16:03,164 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/9271d09d0bec454a8710c5fd815fa3b1 2023-09-06T20:16:03,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/c6d639737f5343cf87f856a46827091c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c6d639737f5343cf87f856a46827091c 2023-09-06T20:16:03,174 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c6d639737f5343cf87f856a46827091c, entries=150, sequenceid=136, filesize=30.3 K 2023-09-06T20:16:03,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/09ffe86814b94b0f8523dbf9c2b14ff4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/09ffe86814b94b0f8523dbf9c2b14ff4 2023-09-06T20:16:03,179 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/09ffe86814b94b0f8523dbf9c2b14ff4, entries=150, sequenceid=136, filesize=11.8 K 2023-09-06T20:16:03,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/9271d09d0bec454a8710c5fd815fa3b1 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/9271d09d0bec454a8710c5fd815fa3b1 2023-09-06T20:16:03,183 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/9271d09d0bec454a8710c5fd815fa3b1, entries=150, sequenceid=136, filesize=11.8 K 2023-09-06T20:16:03,184 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(3022): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=140.89 KB/144270 for 7062ed4ddcd76aee2adae17e574b6875 in 881ms, sequenceid=136, compaction requested=false 2023-09-06T20:16:03,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:03,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:03,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=181 2023-09-06T20:16:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=181 2023-09-06T20:16:03,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=181, resume processing ppid=180 2023-09-06T20:16:03,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=181, ppid=180, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1890 sec 2023-09-06T20:16:03,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=180, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees in 1.1940 sec 2023-09-06T20:16:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:03,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-06T20:16:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:03,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309067880cd69f343424abf1a7db0c38b5f69_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031363547/Put/seqid=0 2023-09-06T20:16:03,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031423555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031423556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031423556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031423559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031423559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031423660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031423660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031423660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031423663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031423663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031423863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031423864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031423864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031423865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031423866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:03,967 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309067880cd69f343424abf1a7db0c38b5f69_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309067880cd69f343424abf1a7db0c38b5f69_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:03,968 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/f89270012a1d40bea84f3d6cf4c58435, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:03,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/f89270012a1d40bea84f3d6cf4c58435 is 175, key is test_row_0/A:col10/1694031363547/Put/seqid=0 2023-09-06T20:16:03,980 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/f89270012a1d40bea84f3d6cf4c58435 2023-09-06T20:16:03,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/bdc513f828bd4d0f8718471ebe06e627 is 50, key is test_row_0/B:col10/1694031363547/Put/seqid=0 2023-09-06T20:16:03,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/bdc513f828bd4d0f8718471ebe06e627 2023-09-06T20:16:03,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/dcbb2c3c534c4df6a358590b6e388a5c is 50, key is test_row_0/C:col10/1694031363547/Put/seqid=0 2023-09-06T20:16:03,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/dcbb2c3c534c4df6a358590b6e388a5c 2023-09-06T20:16:04,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/f89270012a1d40bea84f3d6cf4c58435 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f89270012a1d40bea84f3d6cf4c58435 2023-09-06T20:16:04,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f89270012a1d40bea84f3d6cf4c58435, entries=200, sequenceid=163, filesize=38.8 K 2023-09-06T20:16:04,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/bdc513f828bd4d0f8718471ebe06e627 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/bdc513f828bd4d0f8718471ebe06e627 2023-09-06T20:16:04,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/bdc513f828bd4d0f8718471ebe06e627, entries=150, sequenceid=163, filesize=11.8 K 2023-09-06T20:16:04,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/dcbb2c3c534c4df6a358590b6e388a5c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/dcbb2c3c534c4df6a358590b6e388a5c 2023-09-06T20:16:04,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/dcbb2c3c534c4df6a358590b6e388a5c, entries=150, sequenceid=163, filesize=11.8 K 2023-09-06T20:16:04,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 7062ed4ddcd76aee2adae17e574b6875 in 472ms, sequenceid=163, compaction requested=true 2023-09-06T20:16:04,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:04,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:16:04,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:04,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:16:04,022 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:16:04,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:04,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:16:04,022 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:16:04,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:04,023 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101948 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:16:04,023 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/A is initiating minor compaction (all files) 2023-09-06T20:16:04,024 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/A in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:04,024 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36442 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:16:04,024 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/ebad05b8587642ec8d7e8450e02a0d7b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c6d639737f5343cf87f856a46827091c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f89270012a1d40bea84f3d6cf4c58435] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=99.6 K 2023-09-06T20:16:04,024 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/B is initiating minor compaction (all files) 2023-09-06T20:16:04,024 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:04,024 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/B in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:04,024 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/ebad05b8587642ec8d7e8450e02a0d7b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c6d639737f5343cf87f856a46827091c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f89270012a1d40bea84f3d6cf4c58435] 2023-09-06T20:16:04,024 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/ba2a75fbc530463fa6fcc4d8dd629c4c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/09ffe86814b94b0f8523dbf9c2b14ff4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/bdc513f828bd4d0f8718471ebe06e627] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=35.6 K 2023-09-06T20:16:04,024 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting ebad05b8587642ec8d7e8450e02a0d7b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1694031360622 2023-09-06T20:16:04,025 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting ba2a75fbc530463fa6fcc4d8dd629c4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1694031360622 2023-09-06T20:16:04,025 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting c6d639737f5343cf87f856a46827091c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1694031361744 2023-09-06T20:16:04,025 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 09ffe86814b94b0f8523dbf9c2b14ff4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1694031361744 2023-09-06T20:16:04,025 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting f89270012a1d40bea84f3d6cf4c58435, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1694031362428 2023-09-06T20:16:04,025 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting bdc513f828bd4d0f8718471ebe06e627, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1694031362432 2023-09-06T20:16:04,031 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:04,033 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#B#compaction#565 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:04,033 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/a7bc61f674664d908751ec5fbef1f2bd is 50, key is test_row_0/B:col10/1694031363547/Put/seqid=0 2023-09-06T20:16:04,034 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2023090643a5cd89152d43fdaac4bad49d7ada25_7062ed4ddcd76aee2adae17e574b6875 store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:04,035 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2023090643a5cd89152d43fdaac4bad49d7ada25_7062ed4ddcd76aee2adae17e574b6875, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:04,036 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090643a5cd89152d43fdaac4bad49d7ada25_7062ed4ddcd76aee2adae17e574b6875 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:04,049 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#A#compaction#564 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:04,050 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/afebe6ba93dd4cdc961dd670a02768b4 is 175, key is test_row_0/A:col10/1694031363547/Put/seqid=0 2023-09-06T20:16:04,054 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/a7bc61f674664d908751ec5fbef1f2bd as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/a7bc61f674664d908751ec5fbef1f2bd 2023-09-06T20:16:04,059 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/B of 7062ed4ddcd76aee2adae17e574b6875 into a7bc61f674664d908751ec5fbef1f2bd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:04,059 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/afebe6ba93dd4cdc961dd670a02768b4 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/afebe6ba93dd4cdc961dd670a02768b4 2023-09-06T20:16:04,059 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:04,059 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/B, priority=13, startTime=1694031364022; duration=0sec 2023-09-06T20:16:04,059 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:04,059 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:B 2023-09-06T20:16:04,059 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:16:04,060 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36442 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:16:04,060 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/C is initiating minor compaction (all files) 2023-09-06T20:16:04,060 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/C in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:04,060 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/152d24a29c9e4b319cc6f8e6f3c413fa, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/9271d09d0bec454a8710c5fd815fa3b1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/dcbb2c3c534c4df6a358590b6e388a5c] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=35.6 K 2023-09-06T20:16:04,061 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 152d24a29c9e4b319cc6f8e6f3c413fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1694031360622 2023-09-06T20:16:04,061 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 9271d09d0bec454a8710c5fd815fa3b1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1694031361744 2023-09-06T20:16:04,061 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting dcbb2c3c534c4df6a358590b6e388a5c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1694031362432 2023-09-06T20:16:04,063 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/A of 7062ed4ddcd76aee2adae17e574b6875 into afebe6ba93dd4cdc961dd670a02768b4(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:04,063 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:04,063 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/A, priority=13, startTime=1694031364022; duration=0sec 2023-09-06T20:16:04,064 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:04,064 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:A 2023-09-06T20:16:04,070 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#C#compaction#566 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:04,070 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/9ed31c460ad34a9e8c19201893ec8f7b is 50, key is test_row_0/C:col10/1694031363547/Put/seqid=0 2023-09-06T20:16:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=180 2023-09-06T20:16:04,100 INFO [Thread-3234 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 180 completed 2023-09-06T20:16:04,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:16:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=182, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees 2023-09-06T20:16:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=182 2023-09-06T20:16:04,110 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=182, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:16:04,111 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=182, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:16:04,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=183, ppid=182, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:16:04,114 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/9ed31c460ad34a9e8c19201893ec8f7b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/9ed31c460ad34a9e8c19201893ec8f7b 2023-09-06T20:16:04,118 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/C of 7062ed4ddcd76aee2adae17e574b6875 into 9ed31c460ad34a9e8c19201893ec8f7b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:04,118 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:04,118 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/C, priority=13, startTime=1694031364022; duration=0sec 2023-09-06T20:16:04,118 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:04,118 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:C 2023-09-06T20:16:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:04,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:16:04,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:04,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:04,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:04,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:04,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:04,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:04,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c604178c429a476ea01615ca188065f5_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031364167/Put/seqid=0 2023-09-06T20:16:04,191 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c604178c429a476ea01615ca188065f5_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c604178c429a476ea01615ca188065f5_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:04,192 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/125b5830883c43e2bbff736194044fa8, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:04,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031424190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/125b5830883c43e2bbff736194044fa8 is 175, key is test_row_0/A:col10/1694031364167/Put/seqid=0 2023-09-06T20:16:04,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031424190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031424191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031424192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031424192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,196 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/125b5830883c43e2bbff736194044fa8 2023-09-06T20:16:04,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/528408b3007348a6ad5d1af3614304c6 is 50, key is test_row_0/B:col10/1694031364167/Put/seqid=0 2023-09-06T20:16:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=182 2023-09-06T20:16:04,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/528408b3007348a6ad5d1af3614304c6 2023-09-06T20:16:04,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/add8c040ba3c42af8b904b73fd431e37 is 50, key is test_row_0/C:col10/1694031364167/Put/seqid=0 2023-09-06T20:16:04,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/add8c040ba3c42af8b904b73fd431e37 2023-09-06T20:16:04,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/125b5830883c43e2bbff736194044fa8 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/125b5830883c43e2bbff736194044fa8 2023-09-06T20:16:04,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/125b5830883c43e2bbff736194044fa8, entries=150, sequenceid=177, filesize=30.4 K 2023-09-06T20:16:04,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/528408b3007348a6ad5d1af3614304c6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/528408b3007348a6ad5d1af3614304c6 2023-09-06T20:16:04,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/528408b3007348a6ad5d1af3614304c6, entries=100, sequenceid=177, filesize=9.5 K 2023-09-06T20:16:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/add8c040ba3c42af8b904b73fd431e37 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/add8c040ba3c42af8b904b73fd431e37 2023-09-06T20:16:04,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/add8c040ba3c42af8b904b73fd431e37, entries=100, sequenceid=177, filesize=9.5 K 2023-09-06T20:16:04,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 7062ed4ddcd76aee2adae17e574b6875 in 71ms, sequenceid=177, compaction requested=false 2023-09-06T20:16:04,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:04,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2023-09-06T20:16:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:04,264 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-06T20:16:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:04,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090672e79e1649a64d88b2730821ffc0bd47_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031364191/Put/seqid=0 2023-09-06T20:16:04,288 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090672e79e1649a64d88b2730821ffc0bd47_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090672e79e1649a64d88b2730821ffc0bd47_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:04,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/c9444d58b5a54fd28bccb336a34ff694, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:04,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/c9444d58b5a54fd28bccb336a34ff694 is 175, key is test_row_0/A:col10/1694031364191/Put/seqid=0 2023-09-06T20:16:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:04,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:04,298 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/c9444d58b5a54fd28bccb336a34ff694 2023-09-06T20:16:04,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031424300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031424301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031424302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031424304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/903fc1b0cbe647fc822c1dacb3dbbc25 is 50, key is test_row_0/B:col10/1694031364191/Put/seqid=0 2023-09-06T20:16:04,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031424304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=182 2023-09-06T20:16:04,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031424405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031424406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031424407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031424407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031424409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031424608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031424609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031424610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031424610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031424612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=182 2023-09-06T20:16:04,729 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/903fc1b0cbe647fc822c1dacb3dbbc25 2023-09-06T20:16:04,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/98c64db87ddb4abea2f94bca19e04af6 is 50, key is test_row_0/C:col10/1694031364191/Put/seqid=0 2023-09-06T20:16:04,745 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/98c64db87ddb4abea2f94bca19e04af6 2023-09-06T20:16:04,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/c9444d58b5a54fd28bccb336a34ff694 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c9444d58b5a54fd28bccb336a34ff694 2023-09-06T20:16:04,752 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c9444d58b5a54fd28bccb336a34ff694, entries=150, sequenceid=203, filesize=30.4 K 2023-09-06T20:16:04,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/903fc1b0cbe647fc822c1dacb3dbbc25 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/903fc1b0cbe647fc822c1dacb3dbbc25 2023-09-06T20:16:04,757 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/903fc1b0cbe647fc822c1dacb3dbbc25, entries=150, sequenceid=203, filesize=11.8 K 2023-09-06T20:16:04,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/98c64db87ddb4abea2f94bca19e04af6 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/98c64db87ddb4abea2f94bca19e04af6 2023-09-06T20:16:04,760 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/98c64db87ddb4abea2f94bca19e04af6, entries=150, sequenceid=203, filesize=11.8 K 2023-09-06T20:16:04,761 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 7062ed4ddcd76aee2adae17e574b6875 in 497ms, sequenceid=203, compaction requested=true 2023-09-06T20:16:04,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:04,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:04,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=183 2023-09-06T20:16:04,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=183 2023-09-06T20:16:04,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=183, resume processing ppid=182 2023-09-06T20:16:04,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=183, ppid=182, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 651 msec 2023-09-06T20:16:04,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=182, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees in 662 msec 2023-09-06T20:16:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:04,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-06T20:16:04,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:04,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:04,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:04,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:04,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:04,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:04,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309068b03eee65fac4c49bde0651e9e2d8948_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031364912/Put/seqid=0 2023-09-06T20:16:04,928 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309068b03eee65fac4c49bde0651e9e2d8948_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309068b03eee65fac4c49bde0651e9e2d8948_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:04,932 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/17f16c4ca58840d1876f580184928d80, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:04,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/17f16c4ca58840d1876f580184928d80 is 175, key is test_row_0/A:col10/1694031364912/Put/seqid=0 2023-09-06T20:16:04,937 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/17f16c4ca58840d1876f580184928d80 2023-09-06T20:16:04,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031424936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031424936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031424938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031424938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:04,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031424938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:04,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/46d3211238ff45ac8aee15eb633a4466 is 50, key is test_row_0/B:col10/1694031364912/Put/seqid=0 2023-09-06T20:16:04,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/46d3211238ff45ac8aee15eb633a4466 2023-09-06T20:16:04,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/46a95227a85e465d9e9d888fe115b969 is 50, key is test_row_0/C:col10/1694031364912/Put/seqid=0 2023-09-06T20:16:04,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/46a95227a85e465d9e9d888fe115b969 2023-09-06T20:16:04,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/17f16c4ca58840d1876f580184928d80 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/17f16c4ca58840d1876f580184928d80 2023-09-06T20:16:04,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/17f16c4ca58840d1876f580184928d80, entries=200, sequenceid=216, filesize=38.8 K 2023-09-06T20:16:04,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/46d3211238ff45ac8aee15eb633a4466 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/46d3211238ff45ac8aee15eb633a4466 2023-09-06T20:16:04,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/46d3211238ff45ac8aee15eb633a4466, entries=150, sequenceid=216, filesize=11.8 K 2023-09-06T20:16:04,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/46a95227a85e465d9e9d888fe115b969 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/46a95227a85e465d9e9d888fe115b969 2023-09-06T20:16:04,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/46a95227a85e465d9e9d888fe115b969, entries=150, sequenceid=216, filesize=11.8 K 2023-09-06T20:16:04,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 7062ed4ddcd76aee2adae17e574b6875 in 76ms, sequenceid=216, compaction requested=true 2023-09-06T20:16:04,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:04,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:16:04,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:04,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:16:04,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:04,990 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:16:04,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:16:04,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-06T20:16:04,991 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:16:04,992 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133334 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:16:04,992 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46484 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:16:04,992 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/A is initiating minor compaction (all files) 2023-09-06T20:16:04,992 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/B is initiating minor compaction (all files) 2023-09-06T20:16:04,992 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/B in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:04,992 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/A in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:04,992 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/a7bc61f674664d908751ec5fbef1f2bd, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/528408b3007348a6ad5d1af3614304c6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/903fc1b0cbe647fc822c1dacb3dbbc25, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/46d3211238ff45ac8aee15eb633a4466] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=45.4 K 2023-09-06T20:16:04,992 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/afebe6ba93dd4cdc961dd670a02768b4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/125b5830883c43e2bbff736194044fa8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c9444d58b5a54fd28bccb336a34ff694, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/17f16c4ca58840d1876f580184928d80] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=130.2 K 2023-09-06T20:16:04,992 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:04,992 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/afebe6ba93dd4cdc961dd670a02768b4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/125b5830883c43e2bbff736194044fa8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c9444d58b5a54fd28bccb336a34ff694, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/17f16c4ca58840d1876f580184928d80] 2023-09-06T20:16:04,992 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting a7bc61f674664d908751ec5fbef1f2bd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1694031362432 2023-09-06T20:16:04,992 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting afebe6ba93dd4cdc961dd670a02768b4, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1694031362432 2023-09-06T20:16:04,993 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 528408b3007348a6ad5d1af3614304c6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1694031363552 2023-09-06T20:16:04,993 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 125b5830883c43e2bbff736194044fa8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1694031363552 2023-09-06T20:16:04,993 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 903fc1b0cbe647fc822c1dacb3dbbc25, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1694031364190 2023-09-06T20:16:04,993 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting c9444d58b5a54fd28bccb336a34ff694, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1694031364190 2023-09-06T20:16:04,994 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 46d3211238ff45ac8aee15eb633a4466, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1694031364302 2023-09-06T20:16:04,994 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 17f16c4ca58840d1876f580184928d80, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1694031364298 2023-09-06T20:16:05,003 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#B#compaction#576 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:05,004 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/24657704c57348cd8215bfd0b0a7b723 is 50, key is test_row_0/B:col10/1694031364912/Put/seqid=0 2023-09-06T20:16:05,005 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:05,007 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230906a7c6ec8f80e24de6a5942cb914576c07_7062ed4ddcd76aee2adae17e574b6875 store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:05,008 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230906a7c6ec8f80e24de6a5942cb914576c07_7062ed4ddcd76aee2adae17e574b6875, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:05,009 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906a7c6ec8f80e24de6a5942cb914576c07_7062ed4ddcd76aee2adae17e574b6875 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:05,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:05,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-06T20:16:05,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:05,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:05,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:05,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:05,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:05,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:05,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906f3cb197da08f47f5ba223d896c8b0a83_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031364936/Put/seqid=0 2023-09-06T20:16:05,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031425051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031425051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031425052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031425053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031425053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,064 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906f3cb197da08f47f5ba223d896c8b0a83_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f3cb197da08f47f5ba223d896c8b0a83_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:05,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/9d0b4aac07754a568a79a55f7bc90178, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:05,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/9d0b4aac07754a568a79a55f7bc90178 is 175, key is test_row_0/A:col10/1694031364936/Put/seqid=0 2023-09-06T20:16:05,077 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/9d0b4aac07754a568a79a55f7bc90178 2023-09-06T20:16:05,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/4c52b3ba42e04311803cb76245f793b1 is 50, key is test_row_0/B:col10/1694031364936/Put/seqid=0 2023-09-06T20:16:05,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/4c52b3ba42e04311803cb76245f793b1 2023-09-06T20:16:05,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/cbeaa654f96f49c18a39b8b04c278a16 is 50, key is test_row_0/C:col10/1694031364936/Put/seqid=0 2023-09-06T20:16:05,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031425157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031425157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031425157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031425158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031425162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=182 2023-09-06T20:16:05,206 INFO [Thread-3234 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 182 completed 2023-09-06T20:16:05,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:16:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=184, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees 2023-09-06T20:16:05,210 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=184, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:16:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=184 2023-09-06T20:16:05,211 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=184, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:16:05,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=185, ppid=184, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:16:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=184 2023-09-06T20:16:05,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2023-09-06T20:16:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031425361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:05,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:05,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:05,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(53): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:05,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:05,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031425362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031425363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031425362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031425366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,421 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/24657704c57348cd8215bfd0b0a7b723 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/24657704c57348cd8215bfd0b0a7b723 2023-09-06T20:16:05,424 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#A#compaction#577 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:05,425 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/6510671bd43748e3a4025635073f641d is 175, key is test_row_0/A:col10/1694031364912/Put/seqid=0 2023-09-06T20:16:05,426 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/B of 7062ed4ddcd76aee2adae17e574b6875 into 24657704c57348cd8215bfd0b0a7b723(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:05,426 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:05,427 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/B, priority=12, startTime=1694031364990; duration=0sec 2023-09-06T20:16:05,427 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:05,427 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:B 2023-09-06T20:16:05,427 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:16:05,428 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46484 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:16:05,428 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/C is initiating minor compaction (all files) 2023-09-06T20:16:05,429 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/C in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:05,429 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/9ed31c460ad34a9e8c19201893ec8f7b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/add8c040ba3c42af8b904b73fd431e37, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/98c64db87ddb4abea2f94bca19e04af6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/46a95227a85e465d9e9d888fe115b969] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=45.4 K 2023-09-06T20:16:05,429 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 9ed31c460ad34a9e8c19201893ec8f7b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1694031362432 2023-09-06T20:16:05,430 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting add8c040ba3c42af8b904b73fd431e37, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1694031363552 2023-09-06T20:16:05,430 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 98c64db87ddb4abea2f94bca19e04af6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1694031364190 2023-09-06T20:16:05,438 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 46a95227a85e465d9e9d888fe115b969, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1694031364302 2023-09-06T20:16:05,445 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#C#compaction#581 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:05,445 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/bef224f8b29f4ec8bf11a9169a9349b8 is 50, key is test_row_0/C:col10/1694031364912/Put/seqid=0 2023-09-06T20:16:05,459 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/bef224f8b29f4ec8bf11a9169a9349b8 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/bef224f8b29f4ec8bf11a9169a9349b8 2023-09-06T20:16:05,462 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/C of 7062ed4ddcd76aee2adae17e574b6875 into bef224f8b29f4ec8bf11a9169a9349b8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:05,462 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:05,462 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/C, priority=12, startTime=1694031364990; duration=0sec 2023-09-06T20:16:05,463 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:05,463 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:C 2023-09-06T20:16:05,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/cbeaa654f96f49c18a39b8b04c278a16 2023-09-06T20:16:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=184 2023-09-06T20:16:05,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2023-09-06T20:16:05,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:05,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:05,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:05,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(53): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:05,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:05,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/9d0b4aac07754a568a79a55f7bc90178 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/9d0b4aac07754a568a79a55f7bc90178 2023-09-06T20:16:05,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/9d0b4aac07754a568a79a55f7bc90178, entries=150, sequenceid=242, filesize=30.4 K 2023-09-06T20:16:05,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/4c52b3ba42e04311803cb76245f793b1 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/4c52b3ba42e04311803cb76245f793b1 2023-09-06T20:16:05,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/4c52b3ba42e04311803cb76245f793b1, entries=150, sequenceid=242, filesize=11.8 K 2023-09-06T20:16:05,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/cbeaa654f96f49c18a39b8b04c278a16 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/cbeaa654f96f49c18a39b8b04c278a16 2023-09-06T20:16:05,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/cbeaa654f96f49c18a39b8b04c278a16, entries=150, sequenceid=242, filesize=11.8 K 2023-09-06T20:16:05,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 7062ed4ddcd76aee2adae17e574b6875 in 491ms, sequenceid=242, compaction requested=false 2023-09-06T20:16:05,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:05,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2023-09-06T20:16:05,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:05,669 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2023-09-06T20:16:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:05,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:05,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:05,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:05,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:05,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:05,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:05,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309064ee1f9a299e74366b282fe45c8650373_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031365052/Put/seqid=0 2023-09-06T20:16:05,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031425693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031425694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031425695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031425697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031425698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031425799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031425799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031425799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031425800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031425801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=184 2023-09-06T20:16:05,835 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/6510671bd43748e3a4025635073f641d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6510671bd43748e3a4025635073f641d 2023-09-06T20:16:05,839 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/A of 7062ed4ddcd76aee2adae17e574b6875 into 6510671bd43748e3a4025635073f641d(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:05,839 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:05,839 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/A, priority=12, startTime=1694031364990; duration=0sec 2023-09-06T20:16:05,839 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:05,839 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:A 2023-09-06T20:16:06,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031426002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031426002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031426003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031426004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031426004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,087 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309064ee1f9a299e74366b282fe45c8650373_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309064ee1f9a299e74366b282fe45c8650373_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:06,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/6d4ad27dfb9e4e7c8991499b1ffa2fd2, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/6d4ad27dfb9e4e7c8991499b1ffa2fd2 is 175, key is test_row_0/A:col10/1694031365052/Put/seqid=0 2023-09-06T20:16:06,093 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/6d4ad27dfb9e4e7c8991499b1ffa2fd2 2023-09-06T20:16:06,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/c1d0a505707343a687b29bcb8c7b4fb0 is 50, key is test_row_0/B:col10/1694031365052/Put/seqid=0 2023-09-06T20:16:06,102 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/c1d0a505707343a687b29bcb8c7b4fb0 2023-09-06T20:16:06,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/0f89ddd3f88c4d8bb4948d050a18c962 is 50, key is test_row_0/C:col10/1694031365052/Put/seqid=0 2023-09-06T20:16:06,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031426303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031426305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031426307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031426308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031426308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=184 2023-09-06T20:16:06,515 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/0f89ddd3f88c4d8bb4948d050a18c962 2023-09-06T20:16:06,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/6d4ad27dfb9e4e7c8991499b1ffa2fd2 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6d4ad27dfb9e4e7c8991499b1ffa2fd2 2023-09-06T20:16:06,523 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6d4ad27dfb9e4e7c8991499b1ffa2fd2, entries=100, sequenceid=256, filesize=21.9 K 2023-09-06T20:16:06,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/c1d0a505707343a687b29bcb8c7b4fb0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/c1d0a505707343a687b29bcb8c7b4fb0 2023-09-06T20:16:06,527 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/c1d0a505707343a687b29bcb8c7b4fb0, entries=100, sequenceid=256, filesize=9.5 K 2023-09-06T20:16:06,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/0f89ddd3f88c4d8bb4948d050a18c962 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/0f89ddd3f88c4d8bb4948d050a18c962 2023-09-06T20:16:06,531 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/0f89ddd3f88c4d8bb4948d050a18c962, entries=100, sequenceid=256, filesize=9.5 K 2023-09-06T20:16:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.StoreScanner(982): StoreScanner already closing. There is no need to updateReaders 2023-09-06T20:16:06,532 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 7062ed4ddcd76aee2adae17e574b6875 in 863ms, sequenceid=256, compaction requested=true 2023-09-06T20:16:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=185 2023-09-06T20:16:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=185 2023-09-06T20:16:06,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=185, resume processing ppid=184 2023-09-06T20:16:06,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=185, ppid=184, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3220 sec 2023-09-06T20:16:06,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=184, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees in 1.3270 sec 2023-09-06T20:16:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:06,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-06T20:16:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:06,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c3491e50fc014910be3963550ae34607_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031365696/Put/seqid=0 2023-09-06T20:16:06,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031426816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031426817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031426817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031426818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031426821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,833 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906c3491e50fc014910be3963550ae34607_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c3491e50fc014910be3963550ae34607_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:06,833 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/c9fa613c3f5d45989d1d149e20598092, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:06,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/c9fa613c3f5d45989d1d149e20598092 is 175, key is test_row_0/A:col10/1694031365696/Put/seqid=0 2023-09-06T20:16:06,837 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/c9fa613c3f5d45989d1d149e20598092 2023-09-06T20:16:06,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/5b56ef0094694b6994f8675ad43c4085 is 50, key is test_row_0/B:col10/1694031365696/Put/seqid=0 2023-09-06T20:16:06,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/5b56ef0094694b6994f8675ad43c4085 2023-09-06T20:16:06,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/fc7b3ecc8b4548c5aadaa926364faf1d is 50, key is test_row_0/C:col10/1694031365696/Put/seqid=0 2023-09-06T20:16:06,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031426922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031426922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031426922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031426922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:06,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031426924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031427125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031427125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031427126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031427126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031427128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/fc7b3ecc8b4548c5aadaa926364faf1d 2023-09-06T20:16:07,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/c9fa613c3f5d45989d1d149e20598092 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c9fa613c3f5d45989d1d149e20598092 2023-09-06T20:16:07,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c9fa613c3f5d45989d1d149e20598092, entries=150, sequenceid=283, filesize=30.5 K 2023-09-06T20:16:07,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/5b56ef0094694b6994f8675ad43c4085 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/5b56ef0094694b6994f8675ad43c4085 2023-09-06T20:16:07,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/5b56ef0094694b6994f8675ad43c4085, entries=150, sequenceid=283, filesize=12.0 K 2023-09-06T20:16:07,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/fc7b3ecc8b4548c5aadaa926364faf1d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/fc7b3ecc8b4548c5aadaa926364faf1d 2023-09-06T20:16:07,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/fc7b3ecc8b4548c5aadaa926364faf1d, entries=150, sequenceid=283, filesize=12.0 K 2023-09-06T20:16:07,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 7062ed4ddcd76aee2adae17e574b6875 in 494ms, sequenceid=283, compaction requested=true 2023-09-06T20:16:07,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:07,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:16:07,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:07,305 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:16:07,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:16:07,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:07,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:16:07,305 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:16:07,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:07,307 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 116336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:16:07,307 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46770 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:16:07,307 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/A is initiating minor compaction (all files) 2023-09-06T20:16:07,307 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/B is initiating minor compaction (all files) 2023-09-06T20:16:07,307 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/A in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:07,307 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/B in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:07,307 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6510671bd43748e3a4025635073f641d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/9d0b4aac07754a568a79a55f7bc90178, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6d4ad27dfb9e4e7c8991499b1ffa2fd2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c9fa613c3f5d45989d1d149e20598092] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=113.6 K 2023-09-06T20:16:07,307 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/24657704c57348cd8215bfd0b0a7b723, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/4c52b3ba42e04311803cb76245f793b1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/c1d0a505707343a687b29bcb8c7b4fb0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/5b56ef0094694b6994f8675ad43c4085] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=45.7 K 2023-09-06T20:16:07,307 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:07,307 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6510671bd43748e3a4025635073f641d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/9d0b4aac07754a568a79a55f7bc90178, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6d4ad27dfb9e4e7c8991499b1ffa2fd2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c9fa613c3f5d45989d1d149e20598092] 2023-09-06T20:16:07,307 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 24657704c57348cd8215bfd0b0a7b723, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1694031364302 2023-09-06T20:16:07,308 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6510671bd43748e3a4025635073f641d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1694031364302 2023-09-06T20:16:07,308 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 4c52b3ba42e04311803cb76245f793b1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1694031364936 2023-09-06T20:16:07,308 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9d0b4aac07754a568a79a55f7bc90178, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1694031364936 2023-09-06T20:16:07,308 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting c1d0a505707343a687b29bcb8c7b4fb0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1694031365052 2023-09-06T20:16:07,309 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 5b56ef0094694b6994f8675ad43c4085, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1694031365696 2023-09-06T20:16:07,309 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6d4ad27dfb9e4e7c8991499b1ffa2fd2, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1694031365052 2023-09-06T20:16:07,309 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting c9fa613c3f5d45989d1d149e20598092, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1694031365696 2023-09-06T20:16:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=184 2023-09-06T20:16:07,314 INFO [Thread-3234 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 184 completed 2023-09-06T20:16:07,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:16:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=186, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees 2023-09-06T20:16:07,319 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=186, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:16:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=186 2023-09-06T20:16:07,319 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=186, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:16:07,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=187, ppid=186, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:16:07,324 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#B#compaction#588 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:07,324 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/eb9fc4bdb54d41d8ac2a7dabd1d4e05e is 50, key is test_row_0/B:col10/1694031365696/Put/seqid=0 2023-09-06T20:16:07,325 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:07,346 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230906b52befaab5bf412481b09a38e4ba70ee_7062ed4ddcd76aee2adae17e574b6875 store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:07,348 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230906b52befaab5bf412481b09a38e4ba70ee_7062ed4ddcd76aee2adae17e574b6875, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:07,348 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906b52befaab5bf412481b09a38e4ba70ee_7062ed4ddcd76aee2adae17e574b6875 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:07,382 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#A#compaction#589 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:07,383 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/b9f54d0cb82a416bbb6cd88aa780b5e0 is 175, key is test_row_0/A:col10/1694031365696/Put/seqid=0 2023-09-06T20:16:07,396 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/b9f54d0cb82a416bbb6cd88aa780b5e0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/b9f54d0cb82a416bbb6cd88aa780b5e0 2023-09-06T20:16:07,400 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/A of 7062ed4ddcd76aee2adae17e574b6875 into b9f54d0cb82a416bbb6cd88aa780b5e0(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:07,400 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:07,400 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/A, priority=12, startTime=1694031367305; duration=0sec 2023-09-06T20:16:07,401 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:07,401 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:A 2023-09-06T20:16:07,401 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:16:07,402 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46770 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:16:07,402 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/C is initiating minor compaction (all files) 2023-09-06T20:16:07,402 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/C in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:07,402 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/bef224f8b29f4ec8bf11a9169a9349b8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/cbeaa654f96f49c18a39b8b04c278a16, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/0f89ddd3f88c4d8bb4948d050a18c962, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/fc7b3ecc8b4548c5aadaa926364faf1d] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=45.7 K 2023-09-06T20:16:07,403 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting bef224f8b29f4ec8bf11a9169a9349b8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1694031364302 2023-09-06T20:16:07,404 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting cbeaa654f96f49c18a39b8b04c278a16, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1694031364936 2023-09-06T20:16:07,404 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 0f89ddd3f88c4d8bb4948d050a18c962, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1694031365052 2023-09-06T20:16:07,404 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting fc7b3ecc8b4548c5aadaa926364faf1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1694031365696 2023-09-06T20:16:07,415 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#C#compaction#590 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:07,416 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/280f55df3c094e46a67e5658dcae0018 is 50, key is test_row_0/C:col10/1694031365696/Put/seqid=0 2023-09-06T20:16:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=186 2023-09-06T20:16:07,428 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/280f55df3c094e46a67e5658dcae0018 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/280f55df3c094e46a67e5658dcae0018 2023-09-06T20:16:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:07,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:16:07,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:07,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:07,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:07,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:07,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:07,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:07,433 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/C of 7062ed4ddcd76aee2adae17e574b6875 into 280f55df3c094e46a67e5658dcae0018(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:07,433 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:07,433 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/C, priority=12, startTime=1694031367305; duration=0sec 2023-09-06T20:16:07,433 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:07,433 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:C 2023-09-06T20:16:07,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090684724b492faf4cc1a1755f610b1811c2_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031366820/Put/seqid=0 2023-09-06T20:16:07,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031427450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031427450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031427453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031427453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031427454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-06T20:16:07,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:07,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:07,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:07,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:07,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:07,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031427564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031427564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031427566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031427566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031427566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=186 2023-09-06T20:16:07,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-06T20:16:07,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:07,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:07,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:07,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:07,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:07,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031427768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031427768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031427768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031427768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031427769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:07,772 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/eb9fc4bdb54d41d8ac2a7dabd1d4e05e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/eb9fc4bdb54d41d8ac2a7dabd1d4e05e 2023-09-06T20:16:07,777 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/B of 7062ed4ddcd76aee2adae17e574b6875 into eb9fc4bdb54d41d8ac2a7dabd1d4e05e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:07,777 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:07,777 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/B, priority=12, startTime=1694031367305; duration=0sec 2023-09-06T20:16:07,777 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:07,777 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:B 2023-09-06T20:16:07,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-06T20:16:07,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:07,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:07,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:07,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:07,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:07,844 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090684724b492faf4cc1a1755f610b1811c2_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090684724b492faf4cc1a1755f610b1811c2_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:07,845 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/b592b8aa9daa4055b87d4ad678c9026b, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:07,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/b592b8aa9daa4055b87d4ad678c9026b is 175, key is test_row_0/A:col10/1694031366820/Put/seqid=0 2023-09-06T20:16:07,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=186 2023-09-06T20:16:07,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-06T20:16:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:07,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:07,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031428071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:08,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031428071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:08,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031428072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:08,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031428072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:08,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031428074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:08,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-06T20:16:08,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:08,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:08,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:08,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-06T20:16:08,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:08,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:08,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:08,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,252 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/b592b8aa9daa4055b87d4ad678c9026b 2023-09-06T20:16:08,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/b20a681acce345dc85f5ac0b313b3dca is 50, key is test_row_0/B:col10/1694031366820/Put/seqid=0 2023-09-06T20:16:08,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/b20a681acce345dc85f5ac0b313b3dca 2023-09-06T20:16:08,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/28acbd6e9b674ecab4150aac215ffe17 is 50, key is test_row_0/C:col10/1694031366820/Put/seqid=0 2023-09-06T20:16:08,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-06T20:16:08,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:08,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:08,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:08,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=186 2023-09-06T20:16:08,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-06T20:16:08,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:08,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:08,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:08,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:08,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031428577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:08,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031428577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:08,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031428577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:08,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031428578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:08,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031428579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:08,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/28acbd6e9b674ecab4150aac215ffe17 2023-09-06T20:16:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/b592b8aa9daa4055b87d4ad678c9026b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/b592b8aa9daa4055b87d4ad678c9026b 2023-09-06T20:16:08,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/b592b8aa9daa4055b87d4ad678c9026b, entries=150, sequenceid=295, filesize=30.5 K 2023-09-06T20:16:08,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/b20a681acce345dc85f5ac0b313b3dca as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b20a681acce345dc85f5ac0b313b3dca 2023-09-06T20:16:08,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-06T20:16:08,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:08,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:08,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:08,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b20a681acce345dc85f5ac0b313b3dca, entries=150, sequenceid=295, filesize=12.0 K 2023-09-06T20:16:08,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:08,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/28acbd6e9b674ecab4150aac215ffe17 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/28acbd6e9b674ecab4150aac215ffe17 2023-09-06T20:16:08,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/28acbd6e9b674ecab4150aac215ffe17, entries=150, sequenceid=295, filesize=12.0 K 2023-09-06T20:16:08,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 7062ed4ddcd76aee2adae17e574b6875 in 1272ms, sequenceid=295, compaction requested=false 2023-09-06T20:16:08,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:08,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-06T20:16:08,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:08,850 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-06T20:16:08,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:08,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:08,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:08,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:08,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:08,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:08,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090603b8d103ad5d4038a2b74068fd75514d_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031367449/Put/seqid=0 2023-09-06T20:16:08,867 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090603b8d103ad5d4038a2b74068fd75514d_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090603b8d103ad5d4038a2b74068fd75514d_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:08,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/004d84b6791746f28f6fc69badb75bcb, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:08,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/004d84b6791746f28f6fc69badb75bcb is 175, key is test_row_0/A:col10/1694031367449/Put/seqid=0 2023-09-06T20:16:08,879 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/004d84b6791746f28f6fc69badb75bcb 2023-09-06T20:16:08,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/06427bebd89d49bf8cdc698aa003648c is 50, key is test_row_0/B:col10/1694031367449/Put/seqid=0 2023-09-06T20:16:08,895 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/06427bebd89d49bf8cdc698aa003648c 2023-09-06T20:16:08,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/ade4f73560a747e589138040080f4995 is 50, key is test_row_0/C:col10/1694031367449/Put/seqid=0 2023-09-06T20:16:08,907 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/ade4f73560a747e589138040080f4995 2023-09-06T20:16:08,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/004d84b6791746f28f6fc69badb75bcb as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/004d84b6791746f28f6fc69badb75bcb 2023-09-06T20:16:08,916 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/004d84b6791746f28f6fc69badb75bcb, entries=150, sequenceid=323, filesize=30.5 K 2023-09-06T20:16:08,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/06427bebd89d49bf8cdc698aa003648c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/06427bebd89d49bf8cdc698aa003648c 2023-09-06T20:16:08,920 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/06427bebd89d49bf8cdc698aa003648c, entries=150, sequenceid=323, filesize=12.0 K 2023-09-06T20:16:08,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/ade4f73560a747e589138040080f4995 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ade4f73560a747e589138040080f4995 2023-09-06T20:16:08,924 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ade4f73560a747e589138040080f4995, entries=150, sequenceid=323, filesize=12.0 K 2023-09-06T20:16:08,925 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for 7062ed4ddcd76aee2adae17e574b6875 in 75ms, sequenceid=323, compaction requested=true 2023-09-06T20:16:08,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:08,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:08,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=187 2023-09-06T20:16:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=187 2023-09-06T20:16:08,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=187, resume processing ppid=186 2023-09-06T20:16:08,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=187, ppid=186, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6070 sec 2023-09-06T20:16:08,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=186, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees in 1.6120 sec 2023-09-06T20:16:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=186 2023-09-06T20:16:09,424 INFO [Thread-3234 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 186 completed 2023-09-06T20:16:09,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:16:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=188, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees 2023-09-06T20:16:09,427 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=188, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:16:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=188 2023-09-06T20:16:09,428 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=188, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:16:09,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=189, ppid=188, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:16:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=188 2023-09-06T20:16:09,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2023-09-06T20:16:09,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:09,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:09,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:09,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=189 2023-09-06T20:16:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=189 2023-09-06T20:16:09,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=189, resume processing ppid=188 2023-09-06T20:16:09,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=189, ppid=188, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2023-09-06T20:16:09,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=188, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees in 159 msec 2023-09-06T20:16:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:09,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:16:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:09,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090668af5bd26b8b4f1aa8200f5015e4a29f_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031369605/Put/seqid=0 2023-09-06T20:16:09,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031429632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:09,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031429633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:09,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031429633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:09,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031429634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031429635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=188 2023-09-06T20:16:09,730 INFO [Thread-3234 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 188 completed 2023-09-06T20:16:09,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:16:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=190, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=190, table=TestAcidGuarantees 2023-09-06T20:16:09,733 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=190, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=190, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:16:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=190 2023-09-06T20:16:09,734 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=190, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=190, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:16:09,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=191, ppid=190, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:16:09,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031429737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:09,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031429738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:09,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031429738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031429739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:09,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031429739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=190 2023-09-06T20:16:09,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=191 2023-09-06T20:16:09,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:09,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:09,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:09,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] handler.RSProcedureHandler(53): pid=191 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:09,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=191 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=191 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:09,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031429941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:09,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031429941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:09,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031429942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:09,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031429943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:09,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031429944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,030 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090668af5bd26b8b4f1aa8200f5015e4a29f_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090668af5bd26b8b4f1aa8200f5015e4a29f_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:10,031 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/eca4b362f1144c47b69f5f738d4b2a1e, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:10,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/eca4b362f1144c47b69f5f738d4b2a1e is 175, key is test_row_0/A:col10/1694031369605/Put/seqid=0 2023-09-06T20:16:10,033 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/eca4b362f1144c47b69f5f738d4b2a1e 2023-09-06T20:16:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=190 2023-09-06T20:16:10,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/33bd998fc65840259cf391b0170522bd is 50, key is test_row_0/B:col10/1694031369605/Put/seqid=0 2023-09-06T20:16:10,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=191 2023-09-06T20:16:10,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:10,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:10,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:10,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] handler.RSProcedureHandler(53): pid=191 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:10,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=191 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=191 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:10,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/33bd998fc65840259cf391b0170522bd 2023-09-06T20:16:10,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/ff48bb96f3d34e1994d528a2b633954b is 50, key is test_row_0/C:col10/1694031369605/Put/seqid=0 2023-09-06T20:16:10,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=191 2023-09-06T20:16:10,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:10,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:10,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:10,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] handler.RSProcedureHandler(53): pid=191 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:10,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=191 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=191 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:10,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031430244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031430244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031430246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031430248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031430249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=190 2023-09-06T20:16:10,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=191 2023-09-06T20:16:10,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:10,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:10,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:10,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] handler.RSProcedureHandler(53): pid=191 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:10,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=191 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=191 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:10,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/ff48bb96f3d34e1994d528a2b633954b 2023-09-06T20:16:10,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/eca4b362f1144c47b69f5f738d4b2a1e as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/eca4b362f1144c47b69f5f738d4b2a1e 2023-09-06T20:16:10,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/eca4b362f1144c47b69f5f738d4b2a1e, entries=400, sequenceid=335, filesize=73.0 K 2023-09-06T20:16:10,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/33bd998fc65840259cf391b0170522bd as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/33bd998fc65840259cf391b0170522bd 2023-09-06T20:16:10,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/33bd998fc65840259cf391b0170522bd, entries=150, sequenceid=335, filesize=12.0 K 2023-09-06T20:16:10,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/ff48bb96f3d34e1994d528a2b633954b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ff48bb96f3d34e1994d528a2b633954b 2023-09-06T20:16:10,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ff48bb96f3d34e1994d528a2b633954b, entries=150, sequenceid=335, filesize=12.0 K 2023-09-06T20:16:10,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7062ed4ddcd76aee2adae17e574b6875 in 870ms, sequenceid=335, compaction requested=true 2023-09-06T20:16:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:16:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:10,478 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:16:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:16:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:16:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:10,478 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:16:10,480 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:16:10,480 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 169106 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:16:10,480 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/B is initiating minor compaction (all files) 2023-09-06T20:16:10,480 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/A is initiating minor compaction (all files) 2023-09-06T20:16:10,480 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/B in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:10,480 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/A in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:10,480 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/eb9fc4bdb54d41d8ac2a7dabd1d4e05e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b20a681acce345dc85f5ac0b313b3dca, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/06427bebd89d49bf8cdc698aa003648c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/33bd998fc65840259cf391b0170522bd] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=48.6 K 2023-09-06T20:16:10,480 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/b9f54d0cb82a416bbb6cd88aa780b5e0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/b592b8aa9daa4055b87d4ad678c9026b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/004d84b6791746f28f6fc69badb75bcb, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/eca4b362f1144c47b69f5f738d4b2a1e] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=165.1 K 2023-09-06T20:16:10,480 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:10,480 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/b9f54d0cb82a416bbb6cd88aa780b5e0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/b592b8aa9daa4055b87d4ad678c9026b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/004d84b6791746f28f6fc69badb75bcb, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/eca4b362f1144c47b69f5f738d4b2a1e] 2023-09-06T20:16:10,480 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting eb9fc4bdb54d41d8ac2a7dabd1d4e05e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1694031365696 2023-09-06T20:16:10,480 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting b9f54d0cb82a416bbb6cd88aa780b5e0, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1694031365696 2023-09-06T20:16:10,480 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b20a681acce345dc85f5ac0b313b3dca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1694031366814 2023-09-06T20:16:10,481 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting b592b8aa9daa4055b87d4ad678c9026b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1694031366814 2023-09-06T20:16:10,481 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 06427bebd89d49bf8cdc698aa003648c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1694031367448 2023-09-06T20:16:10,481 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 004d84b6791746f28f6fc69badb75bcb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1694031367448 2023-09-06T20:16:10,481 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 33bd998fc65840259cf391b0170522bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1694031369605 2023-09-06T20:16:10,481 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting eca4b362f1144c47b69f5f738d4b2a1e, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1694031369596 2023-09-06T20:16:10,487 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:10,488 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#B#compaction#600 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:10,488 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/c722acadb4304e2e9b7c5cd60144ba1a is 50, key is test_row_0/B:col10/1694031369605/Put/seqid=0 2023-09-06T20:16:10,490 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309061f774e03f8c049c686bd50c8ff38d540_7062ed4ddcd76aee2adae17e574b6875 store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:10,492 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309061f774e03f8c049c686bd50c8ff38d540_7062ed4ddcd76aee2adae17e574b6875, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:10,492 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309061f774e03f8c049c686bd50c8ff38d540_7062ed4ddcd76aee2adae17e574b6875 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:10,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=191 2023-09-06T20:16:10,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:10,498 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-06T20:16:10,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:10,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:10,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:10,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:10,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:10,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:10,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906678856ffc8eb4c1496d63cf7db5f6754_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031369633/Put/seqid=0 2023-09-06T20:16:10,534 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/c722acadb4304e2e9b7c5cd60144ba1a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/c722acadb4304e2e9b7c5cd60144ba1a 2023-09-06T20:16:10,539 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/B of 7062ed4ddcd76aee2adae17e574b6875 into c722acadb4304e2e9b7c5cd60144ba1a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:10,539 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:10,539 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/B, priority=12, startTime=1694031370478; duration=0sec 2023-09-06T20:16:10,539 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:10,539 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:B 2023-09-06T20:16:10,539 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-06T20:16:10,540 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-06T20:16:10,540 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/C is initiating minor compaction (all files) 2023-09-06T20:16:10,541 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/C in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:10,541 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/280f55df3c094e46a67e5658dcae0018, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/28acbd6e9b674ecab4150aac215ffe17, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ade4f73560a747e589138040080f4995, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ff48bb96f3d34e1994d528a2b633954b] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=48.6 K 2023-09-06T20:16:10,541 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 280f55df3c094e46a67e5658dcae0018, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1694031365696 2023-09-06T20:16:10,541 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 28acbd6e9b674ecab4150aac215ffe17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1694031366814 2023-09-06T20:16:10,542 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting ade4f73560a747e589138040080f4995, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1694031367448 2023-09-06T20:16:10,542 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting ff48bb96f3d34e1994d528a2b633954b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1694031369605 2023-09-06T20:16:10,549 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#C#compaction#603 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:10,550 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/26c160f39a3d497db0e3adf76900706a is 50, key is test_row_0/C:col10/1694031369605/Put/seqid=0 2023-09-06T20:16:10,558 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/26c160f39a3d497db0e3adf76900706a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/26c160f39a3d497db0e3adf76900706a 2023-09-06T20:16:10,562 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/C of 7062ed4ddcd76aee2adae17e574b6875 into 26c160f39a3d497db0e3adf76900706a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:10,562 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:10,562 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/C, priority=12, startTime=1694031370478; duration=0sec 2023-09-06T20:16:10,562 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:10,562 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:C 2023-09-06T20:16:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:10,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:10,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031430756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031430756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031430756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031430759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031430759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=190 2023-09-06T20:16:10,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031430860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031430861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031430861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031430863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031430866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:10,921 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#A#compaction#601 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:10,921 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/a8734cb9e3c444778e6b70d8653ec991 is 175, key is test_row_0/A:col10/1694031369605/Put/seqid=0 2023-09-06T20:16:10,942 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/a8734cb9e3c444778e6b70d8653ec991 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/a8734cb9e3c444778e6b70d8653ec991 2023-09-06T20:16:10,948 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906678856ffc8eb4c1496d63cf7db5f6754_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906678856ffc8eb4c1496d63cf7db5f6754_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:10,948 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/A of 7062ed4ddcd76aee2adae17e574b6875 into a8734cb9e3c444778e6b70d8653ec991(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:10,948 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:10,948 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/A, priority=12, startTime=1694031370478; duration=0sec 2023-09-06T20:16:10,948 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:10,948 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:A 2023-09-06T20:16:10,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/83d07452fdb44137877de30b6aae1033, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:10,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/83d07452fdb44137877de30b6aae1033 is 175, key is test_row_0/A:col10/1694031369633/Put/seqid=0 2023-09-06T20:16:11,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031431065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031431065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031431066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031431068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031431068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031431367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031431369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031431369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031431370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,372 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/83d07452fdb44137877de30b6aae1033 2023-09-06T20:16:11,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031431372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/4d91d3a619e74fb9ae196f93128c738a is 50, key is test_row_0/B:col10/1694031369633/Put/seqid=0 2023-09-06T20:16:11,790 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/4d91d3a619e74fb9ae196f93128c738a 2023-09-06T20:16:11,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/bf4a067dbafc41e893a572f82d3280b2 is 50, key is test_row_0/C:col10/1694031369633/Put/seqid=0 2023-09-06T20:16:11,803 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/bf4a067dbafc41e893a572f82d3280b2 2023-09-06T20:16:11,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/83d07452fdb44137877de30b6aae1033 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/83d07452fdb44137877de30b6aae1033 2023-09-06T20:16:11,812 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/83d07452fdb44137877de30b6aae1033, entries=150, sequenceid=360, filesize=30.5 K 2023-09-06T20:16:11,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/4d91d3a619e74fb9ae196f93128c738a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/4d91d3a619e74fb9ae196f93128c738a 2023-09-06T20:16:11,823 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/4d91d3a619e74fb9ae196f93128c738a, entries=150, sequenceid=360, filesize=12.0 K 2023-09-06T20:16:11,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/bf4a067dbafc41e893a572f82d3280b2 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/bf4a067dbafc41e893a572f82d3280b2 2023-09-06T20:16:11,827 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/bf4a067dbafc41e893a572f82d3280b2, entries=150, sequenceid=360, filesize=12.0 K 2023-09-06T20:16:11,828 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 7062ed4ddcd76aee2adae17e574b6875 in 1330ms, sequenceid=360, compaction requested=false 2023-09-06T20:16:11,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:11,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:11,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=191 2023-09-06T20:16:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=191 2023-09-06T20:16:11,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=191, resume processing ppid=190 2023-09-06T20:16:11,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=191, ppid=190, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0950 sec 2023-09-06T20:16:11,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=190, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=190, table=TestAcidGuarantees in 2.1010 sec 2023-09-06T20:16:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=190 2023-09-06T20:16:11,843 INFO [Thread-3234 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 190 completed 2023-09-06T20:16:11,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:16:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=192, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=192, table=TestAcidGuarantees 2023-09-06T20:16:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=192 2023-09-06T20:16:11,846 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=192, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=192, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:16:11,847 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=192, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=192, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:16:11,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=193, ppid=192, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:16:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:11,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-06T20:16:11,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:11,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:11,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:11,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:11,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:11,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:11,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309060881824809c94b24b2de25a2d0a0891c_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031370758/Put/seqid=0 2023-09-06T20:16:11,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031431896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031431897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031431897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031431900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031431900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=192 2023-09-06T20:16:12,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2023-09-06T20:16:12,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031432001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:12,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] handler.RSProcedureHandler(53): pid=193 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=193 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031432001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=193 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031432002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031432003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031432003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=192 2023-09-06T20:16:12,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2023-09-06T20:16:12,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:12,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] handler.RSProcedureHandler(53): pid=193 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=193 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=193 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031432202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031432203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031432204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031432206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031432219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,288 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309060881824809c94b24b2de25a2d0a0891c_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309060881824809c94b24b2de25a2d0a0891c_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:12,289 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/f4075035286b48fc9fa225ca5bedf556, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:12,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/f4075035286b48fc9fa225ca5bedf556 is 175, key is test_row_0/A:col10/1694031370758/Put/seqid=0 2023-09-06T20:16:12,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2023-09-06T20:16:12,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:12,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] handler.RSProcedureHandler(53): pid=193 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=193 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=193 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=192 2023-09-06T20:16:12,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2023-09-06T20:16:12,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:12,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] handler.RSProcedureHandler(53): pid=193 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=193 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=193 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031432505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031432506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031432506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031432524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031432525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:12,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2023-09-06T20:16:12,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:12,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] handler.RSProcedureHandler(53): pid=193 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=193 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=193 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:12,693 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/f4075035286b48fc9fa225ca5bedf556 2023-09-06T20:16:12,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/0586a50467d54f188475c6ec38b9961d is 50, key is test_row_0/B:col10/1694031370758/Put/seqid=0 2023-09-06T20:16:12,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/0586a50467d54f188475c6ec38b9961d 2023-09-06T20:16:12,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/49de05ec2cf9494bbde81ac6d0a26807 is 50, key is test_row_0/C:col10/1694031370758/Put/seqid=0 2023-09-06T20:16:12,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/49de05ec2cf9494bbde81ac6d0a26807 2023-09-06T20:16:12,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/f4075035286b48fc9fa225ca5bedf556 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f4075035286b48fc9fa225ca5bedf556 2023-09-06T20:16:12,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f4075035286b48fc9fa225ca5bedf556, entries=150, sequenceid=376, filesize=30.5 K 2023-09-06T20:16:12,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/0586a50467d54f188475c6ec38b9961d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/0586a50467d54f188475c6ec38b9961d 2023-09-06T20:16:12,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/0586a50467d54f188475c6ec38b9961d, entries=150, sequenceid=376, filesize=12.0 K 2023-09-06T20:16:12,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/49de05ec2cf9494bbde81ac6d0a26807 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/49de05ec2cf9494bbde81ac6d0a26807 2023-09-06T20:16:12,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/49de05ec2cf9494bbde81ac6d0a26807, entries=150, sequenceid=376, filesize=12.0 K 2023-09-06T20:16:12,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 7062ed4ddcd76aee2adae17e574b6875 in 865ms, sequenceid=376, compaction requested=true 2023-09-06T20:16:12,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:12,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:16:12,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:12,739 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:16:12,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:16:12,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:12,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:16:12,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:12,739 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:16:12,740 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94464 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:16:12,740 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37602 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:16:12,741 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/A is initiating minor compaction (all files) 2023-09-06T20:16:12,741 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/B is initiating minor compaction (all files) 2023-09-06T20:16:12,741 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/A in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,741 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/B in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,741 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/a8734cb9e3c444778e6b70d8653ec991, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/83d07452fdb44137877de30b6aae1033, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f4075035286b48fc9fa225ca5bedf556] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=92.3 K 2023-09-06T20:16:12,741 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/c722acadb4304e2e9b7c5cd60144ba1a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/4d91d3a619e74fb9ae196f93128c738a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/0586a50467d54f188475c6ec38b9961d] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=36.7 K 2023-09-06T20:16:12,741 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,741 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/a8734cb9e3c444778e6b70d8653ec991, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/83d07452fdb44137877de30b6aae1033, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f4075035286b48fc9fa225ca5bedf556] 2023-09-06T20:16:12,741 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting c722acadb4304e2e9b7c5cd60144ba1a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1694031369605 2023-09-06T20:16:12,742 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting a8734cb9e3c444778e6b70d8653ec991, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1694031369605 2023-09-06T20:16:12,742 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 4d91d3a619e74fb9ae196f93128c738a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1694031369631 2023-09-06T20:16:12,742 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 83d07452fdb44137877de30b6aae1033, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1694031369631 2023-09-06T20:16:12,742 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 0586a50467d54f188475c6ec38b9961d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1694031370758 2023-09-06T20:16:12,742 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting f4075035286b48fc9fa225ca5bedf556, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1694031370758 2023-09-06T20:16:12,749 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#B#compaction#609 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:12,749 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/d72225a9231a4a328bb04ace266ec403 is 50, key is test_row_0/B:col10/1694031370758/Put/seqid=0 2023-09-06T20:16:12,750 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:12,751 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230906adcdd7c55b734f049f7c4f36112e0cd8_7062ed4ddcd76aee2adae17e574b6875 store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:12,752 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230906adcdd7c55b734f049f7c4f36112e0cd8_7062ed4ddcd76aee2adae17e574b6875, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:12,752 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906adcdd7c55b734f049f7c4f36112e0cd8_7062ed4ddcd76aee2adae17e574b6875 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:12,762 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/d72225a9231a4a328bb04ace266ec403 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d72225a9231a4a328bb04ace266ec403 2023-09-06T20:16:12,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2023-09-06T20:16:12,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,765 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-06T20:16:12,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:12,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:12,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:12,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:12,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:12,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:12,768 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/B of 7062ed4ddcd76aee2adae17e574b6875 into d72225a9231a4a328bb04ace266ec403(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:12,768 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:12,768 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/B, priority=13, startTime=1694031372739; duration=0sec 2023-09-06T20:16:12,768 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:12,768 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:B 2023-09-06T20:16:12,769 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:16:12,770 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37602 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:16:12,770 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/C is initiating minor compaction (all files) 2023-09-06T20:16:12,770 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/C in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,770 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/26c160f39a3d497db0e3adf76900706a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/bf4a067dbafc41e893a572f82d3280b2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/49de05ec2cf9494bbde81ac6d0a26807] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=36.7 K 2023-09-06T20:16:12,771 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 26c160f39a3d497db0e3adf76900706a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1694031369605 2023-09-06T20:16:12,771 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#A#compaction#610 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:12,771 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting bf4a067dbafc41e893a572f82d3280b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1694031369631 2023-09-06T20:16:12,771 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 49de05ec2cf9494bbde81ac6d0a26807, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1694031370758 2023-09-06T20:16:12,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906240bf7aa93934ceeb2949d62ff7ca61f_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031371898/Put/seqid=0 2023-09-06T20:16:12,771 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/fbabfe6c9ec34ec78bcb8bb914f0918f is 175, key is test_row_0/A:col10/1694031370758/Put/seqid=0 2023-09-06T20:16:12,776 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#C#compaction#612 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:12,777 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/46cc145beb944b33b9f62975c2e8de77 is 50, key is test_row_0/C:col10/1694031370758/Put/seqid=0 2023-09-06T20:16:12,780 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/fbabfe6c9ec34ec78bcb8bb914f0918f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/fbabfe6c9ec34ec78bcb8bb914f0918f 2023-09-06T20:16:12,782 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906240bf7aa93934ceeb2949d62ff7ca61f_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906240bf7aa93934ceeb2949d62ff7ca61f_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/fa4087b6f06a4f8b87931a6cf3fe94d8, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/fa4087b6f06a4f8b87931a6cf3fe94d8 is 175, key is test_row_0/A:col10/1694031371898/Put/seqid=0 2023-09-06T20:16:12,786 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/A of 7062ed4ddcd76aee2adae17e574b6875 into fbabfe6c9ec34ec78bcb8bb914f0918f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:12,786 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:12,786 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/A, priority=13, startTime=1694031372739; duration=0sec 2023-09-06T20:16:12,786 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/46cc145beb944b33b9f62975c2e8de77 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/46cc145beb944b33b9f62975c2e8de77 2023-09-06T20:16:12,786 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:12,786 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:A 2023-09-06T20:16:12,790 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/C of 7062ed4ddcd76aee2adae17e574b6875 into 46cc145beb944b33b9f62975c2e8de77(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:12,790 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:12,790 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/C, priority=13, startTime=1694031372739; duration=0sec 2023-09-06T20:16:12,790 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:12,790 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:C 2023-09-06T20:16:12,792 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=400, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/fa4087b6f06a4f8b87931a6cf3fe94d8 2023-09-06T20:16:12,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/7174c6e9080a493bad26b7098175fec0 is 50, key is test_row_0/B:col10/1694031371898/Put/seqid=0 2023-09-06T20:16:12,805 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/7174c6e9080a493bad26b7098175fec0 2023-09-06T20:16:12,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/3931c298b80341b8989afb9437bf5323 is 50, key is test_row_0/C:col10/1694031371898/Put/seqid=0 2023-09-06T20:16:12,816 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/3931c298b80341b8989afb9437bf5323 2023-09-06T20:16:12,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/fa4087b6f06a4f8b87931a6cf3fe94d8 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/fa4087b6f06a4f8b87931a6cf3fe94d8 2023-09-06T20:16:12,823 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/fa4087b6f06a4f8b87931a6cf3fe94d8, entries=150, sequenceid=400, filesize=30.5 K 2023-09-06T20:16:12,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/7174c6e9080a493bad26b7098175fec0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7174c6e9080a493bad26b7098175fec0 2023-09-06T20:16:12,827 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7174c6e9080a493bad26b7098175fec0, entries=150, sequenceid=400, filesize=12.0 K 2023-09-06T20:16:12,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/3931c298b80341b8989afb9437bf5323 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/3931c298b80341b8989afb9437bf5323 2023-09-06T20:16:12,832 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/3931c298b80341b8989afb9437bf5323, entries=150, sequenceid=400, filesize=12.0 K 2023-09-06T20:16:12,832 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 7062ed4ddcd76aee2adae17e574b6875 in 67ms, sequenceid=400, compaction requested=false 2023-09-06T20:16:12,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:12,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:12,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=193 2023-09-06T20:16:12,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=193 2023-09-06T20:16:12,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=193, resume processing ppid=192 2023-09-06T20:16:12,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=193, ppid=192, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 986 msec 2023-09-06T20:16:12,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=192, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=192, table=TestAcidGuarantees in 991 msec 2023-09-06T20:16:12,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=192 2023-09-06T20:16:12,949 INFO [Thread-3234 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 192 completed 2023-09-06T20:16:12,951 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:16:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=194, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=194, table=TestAcidGuarantees 2023-09-06T20:16:12,953 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=194, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=194, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:16:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=194 2023-09-06T20:16:12,953 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=194, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=194, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:16:12,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=195, ppid=194, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:16:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:13,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:16:13,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:13,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:13,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:13,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:13,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:13,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=194 2023-09-06T20:16:13,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090608f823b589f34171919f781e35966017_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031373049/Put/seqid=0 2023-09-06T20:16:13,066 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090608f823b589f34171919f781e35966017_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090608f823b589f34171919f781e35966017_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:13,067 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/f9c224211f444052998b8ea42785412f, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:13,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/f9c224211f444052998b8ea42785412f is 175, key is test_row_0/A:col10/1694031373049/Put/seqid=0 2023-09-06T20:16:13,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031433075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031433078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031433080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031433079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031433080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=195 2023-09-06T20:16:13,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:13,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] handler.RSProcedureHandler(53): pid=195 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=195 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=195 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031433181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031433184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031433185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031433185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031433185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=194 2023-09-06T20:16:13,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=195 2023-09-06T20:16:13,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:13,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] handler.RSProcedureHandler(53): pid=195 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=195 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=195 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031433384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031433387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031433388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031433388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031433389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=195 2023-09-06T20:16:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] handler.RSProcedureHandler(53): pid=195 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=195 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=195 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,493 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=414, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/f9c224211f444052998b8ea42785412f 2023-09-06T20:16:13,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/7db4ce67d65646c5afba469e69f01d00 is 50, key is test_row_0/B:col10/1694031373049/Put/seqid=0 2023-09-06T20:16:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=194 2023-09-06T20:16:13,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=195 2023-09-06T20:16:13,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:13,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] handler.RSProcedureHandler(53): pid=195 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=195 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=195 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031433686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031433689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031433691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031433692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031433694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:13,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=195 2023-09-06T20:16:13,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:13,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] handler.RSProcedureHandler(53): pid=195 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=195 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=195 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=195 2023-09-06T20:16:13,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:13,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] handler.RSProcedureHandler(53): pid=195 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=195 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=195 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:13,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/7db4ce67d65646c5afba469e69f01d00 2023-09-06T20:16:13,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/f74c2135ef8c4f71956a283cbaf2c769 is 50, key is test_row_0/C:col10/1694031373049/Put/seqid=0 2023-09-06T20:16:13,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/f74c2135ef8c4f71956a283cbaf2c769 2023-09-06T20:16:13,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/f9c224211f444052998b8ea42785412f as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f9c224211f444052998b8ea42785412f 2023-09-06T20:16:13,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f9c224211f444052998b8ea42785412f, entries=350, sequenceid=414, filesize=64.5 K 2023-09-06T20:16:13,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/7db4ce67d65646c5afba469e69f01d00 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7db4ce67d65646c5afba469e69f01d00 2023-09-06T20:16:13,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7db4ce67d65646c5afba469e69f01d00, entries=150, sequenceid=414, filesize=12.0 K 2023-09-06T20:16:13,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/f74c2135ef8c4f71956a283cbaf2c769 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/f74c2135ef8c4f71956a283cbaf2c769 2023-09-06T20:16:13,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/f74c2135ef8c4f71956a283cbaf2c769, entries=150, sequenceid=414, filesize=12.0 K 2023-09-06T20:16:13,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 7062ed4ddcd76aee2adae17e574b6875 in 888ms, sequenceid=414, compaction requested=true 2023-09-06T20:16:13,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:13,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:16:13,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:13,939 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:16:13,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:16:13,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:13,939 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:16:13,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:16:13,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:13,940 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129334 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:16:13,940 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/A is initiating minor compaction (all files) 2023-09-06T20:16:13,941 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37704 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:16:13,941 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/A in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,941 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/B is initiating minor compaction (all files) 2023-09-06T20:16:13,941 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/B in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,941 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/fbabfe6c9ec34ec78bcb8bb914f0918f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/fa4087b6f06a4f8b87931a6cf3fe94d8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f9c224211f444052998b8ea42785412f] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=126.3 K 2023-09-06T20:16:13,941 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d72225a9231a4a328bb04ace266ec403, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7174c6e9080a493bad26b7098175fec0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7db4ce67d65646c5afba469e69f01d00] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=36.8 K 2023-09-06T20:16:13,941 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,941 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/fbabfe6c9ec34ec78bcb8bb914f0918f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/fa4087b6f06a4f8b87931a6cf3fe94d8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f9c224211f444052998b8ea42785412f] 2023-09-06T20:16:13,941 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting d72225a9231a4a328bb04ace266ec403, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1694031370758 2023-09-06T20:16:13,941 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting fbabfe6c9ec34ec78bcb8bb914f0918f, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1694031370758 2023-09-06T20:16:13,941 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 7174c6e9080a493bad26b7098175fec0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1694031371896 2023-09-06T20:16:13,942 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting fa4087b6f06a4f8b87931a6cf3fe94d8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1694031371896 2023-09-06T20:16:13,942 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 7db4ce67d65646c5afba469e69f01d00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1694031373047 2023-09-06T20:16:13,942 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting f9c224211f444052998b8ea42785412f, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1694031373028 2023-09-06T20:16:13,948 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#B#compaction#618 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:13,948 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/2846ab78a0c741e289658017bd621a17 is 50, key is test_row_0/B:col10/1694031373049/Put/seqid=0 2023-09-06T20:16:13,949 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:13,953 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2023090673993bfa9ffb4c25bdc9f60b8e9e4934_7062ed4ddcd76aee2adae17e574b6875 store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:13,955 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2023090673993bfa9ffb4c25bdc9f60b8e9e4934_7062ed4ddcd76aee2adae17e574b6875, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:13,955 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090673993bfa9ffb4c25bdc9f60b8e9e4934_7062ed4ddcd76aee2adae17e574b6875 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:13,960 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#A#compaction#619 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:13,961 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/2c7a0a68a96f4960a4135783b17b5c8c is 175, key is test_row_0/A:col10/1694031373049/Put/seqid=0 2023-09-06T20:16:13,962 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/2846ab78a0c741e289658017bd621a17 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/2846ab78a0c741e289658017bd621a17 2023-09-06T20:16:13,966 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/B of 7062ed4ddcd76aee2adae17e574b6875 into 2846ab78a0c741e289658017bd621a17(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:13,967 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:13,967 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/B, priority=13, startTime=1694031373939; duration=0sec 2023-09-06T20:16:13,967 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:13,967 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:B 2023-09-06T20:16:13,967 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:16:13,968 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37704 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:16:13,968 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/C is initiating minor compaction (all files) 2023-09-06T20:16:13,968 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/C in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:13,968 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/46cc145beb944b33b9f62975c2e8de77, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/3931c298b80341b8989afb9437bf5323, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/f74c2135ef8c4f71956a283cbaf2c769] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=36.8 K 2023-09-06T20:16:13,969 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 46cc145beb944b33b9f62975c2e8de77, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1694031370758 2023-09-06T20:16:13,969 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 3931c298b80341b8989afb9437bf5323, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1694031371896 2023-09-06T20:16:13,969 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting f74c2135ef8c4f71956a283cbaf2c769, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1694031373047 2023-09-06T20:16:13,975 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#C#compaction#620 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:13,975 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/e4d7aae9c66849c788e07b60d66826d7 is 50, key is test_row_0/C:col10/1694031373049/Put/seqid=0 2023-09-06T20:16:13,984 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/2c7a0a68a96f4960a4135783b17b5c8c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/2c7a0a68a96f4960a4135783b17b5c8c 2023-09-06T20:16:13,987 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/A of 7062ed4ddcd76aee2adae17e574b6875 into 2c7a0a68a96f4960a4135783b17b5c8c(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:13,987 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:13,987 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/A, priority=13, startTime=1694031373939; duration=0sec 2023-09-06T20:16:13,988 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:13,988 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:A 2023-09-06T20:16:13,998 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/e4d7aae9c66849c788e07b60d66826d7 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/e4d7aae9c66849c788e07b60d66826d7 2023-09-06T20:16:14,003 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/C of 7062ed4ddcd76aee2adae17e574b6875 into e4d7aae9c66849c788e07b60d66826d7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:14,003 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:14,003 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/C, priority=13, startTime=1694031373939; duration=0sec 2023-09-06T20:16:14,003 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:14,003 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:C 2023-09-06T20:16:14,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=195 2023-09-06T20:16:14,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:14,022 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-06T20:16:14,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:14,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:14,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:14,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:14,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:14,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:14,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906b8dff0da9a4b4960a201a760b24ff6d5_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031373067/Put/seqid=0 2023-09-06T20:16:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=194 2023-09-06T20:16:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:14,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:14,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031434201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031434201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031434202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031434203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031434207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031434304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031434304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031434305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031434306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031434308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,435 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906b8dff0da9a4b4960a201a760b24ff6d5_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906b8dff0da9a4b4960a201a760b24ff6d5_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/6ac777260bc848bc87085886ec34e748, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/6ac777260bc848bc87085886ec34e748 is 175, key is test_row_0/A:col10/1694031373067/Put/seqid=0 2023-09-06T20:16:14,462 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=443, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/6ac777260bc848bc87085886ec34e748 2023-09-06T20:16:14,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/d6bee103de644ecca98c5015e87815cf is 50, key is test_row_0/B:col10/1694031373067/Put/seqid=0 2023-09-06T20:16:14,475 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/d6bee103de644ecca98c5015e87815cf 2023-09-06T20:16:14,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/7dec6b6399dd41bbb2d523b703c50a36 is 50, key is test_row_0/C:col10/1694031373067/Put/seqid=0 2023-09-06T20:16:14,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031434507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031434508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031434508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031434509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031434512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031434809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031434810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031434810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031434812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031434813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:14,892 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/7dec6b6399dd41bbb2d523b703c50a36 2023-09-06T20:16:14,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/6ac777260bc848bc87085886ec34e748 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6ac777260bc848bc87085886ec34e748 2023-09-06T20:16:14,901 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6ac777260bc848bc87085886ec34e748, entries=150, sequenceid=443, filesize=30.5 K 2023-09-06T20:16:14,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/d6bee103de644ecca98c5015e87815cf as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d6bee103de644ecca98c5015e87815cf 2023-09-06T20:16:14,905 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d6bee103de644ecca98c5015e87815cf, entries=150, sequenceid=443, filesize=12.0 K 2023-09-06T20:16:14,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/7dec6b6399dd41bbb2d523b703c50a36 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/7dec6b6399dd41bbb2d523b703c50a36 2023-09-06T20:16:14,908 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/7dec6b6399dd41bbb2d523b703c50a36, entries=150, sequenceid=443, filesize=12.0 K 2023-09-06T20:16:14,909 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 7062ed4ddcd76aee2adae17e574b6875 in 888ms, sequenceid=443, compaction requested=false 2023-09-06T20:16:14,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:14,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:14,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=195 2023-09-06T20:16:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=195 2023-09-06T20:16:14,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=195, resume processing ppid=194 2023-09-06T20:16:14,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=195, ppid=194, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9560 sec 2023-09-06T20:16:14,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=194, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=194, table=TestAcidGuarantees in 1.9610 sec 2023-09-06T20:16:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=194 2023-09-06T20:16:15,057 INFO [Thread-3234 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 194 completed 2023-09-06T20:16:15,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:16:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=196, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=196, table=TestAcidGuarantees 2023-09-06T20:16:15,061 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=196, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=196, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:16:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=196 2023-09-06T20:16:15,061 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=196, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=196, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:16:15,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=197, ppid=196, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:16:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=196 2023-09-06T20:16:15,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=197 2023-09-06T20:16:15,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:15,214 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2023-09-06T20:16:15,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:15,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:15,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:15,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:15,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:15,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:15,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090656e54114f89145b3b1b280ae8dac3d9d_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031374197/Put/seqid=0 2023-09-06T20:16:15,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:15,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031435335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031435336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031435339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031435339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031435343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=196 2023-09-06T20:16:15,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031435440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031435440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031435441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031435442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031435444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,636 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023090656e54114f89145b3b1b280ae8dac3d9d_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090656e54114f89145b3b1b280ae8dac3d9d_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:15,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/6af8a31a976d46308d3bb43ee0e6b589, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:15,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/6af8a31a976d46308d3bb43ee0e6b589 is 175, key is test_row_0/A:col10/1694031374197/Put/seqid=0 2023-09-06T20:16:15,642 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=453, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/6af8a31a976d46308d3bb43ee0e6b589 2023-09-06T20:16:15,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031435642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031435643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031435645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031435646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031435646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/cbc851d8edc1489682f849ba55e6175b is 50, key is test_row_0/B:col10/1694031374197/Put/seqid=0 2023-09-06T20:16:15,656 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/cbc851d8edc1489682f849ba55e6175b 2023-09-06T20:16:15,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/d4fcb082db5d4f81a84266962e575f3c is 50, key is test_row_0/C:col10/1694031374197/Put/seqid=0 2023-09-06T20:16:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=196 2023-09-06T20:16:15,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031435944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031435946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031435950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031435950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:15,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031435951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:16,071 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/d4fcb082db5d4f81a84266962e575f3c 2023-09-06T20:16:16,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/6af8a31a976d46308d3bb43ee0e6b589 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6af8a31a976d46308d3bb43ee0e6b589 2023-09-06T20:16:16,079 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6af8a31a976d46308d3bb43ee0e6b589, entries=150, sequenceid=453, filesize=30.5 K 2023-09-06T20:16:16,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/cbc851d8edc1489682f849ba55e6175b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/cbc851d8edc1489682f849ba55e6175b 2023-09-06T20:16:16,084 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/cbc851d8edc1489682f849ba55e6175b, entries=150, sequenceid=453, filesize=12.0 K 2023-09-06T20:16:16,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/d4fcb082db5d4f81a84266962e575f3c as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/d4fcb082db5d4f81a84266962e575f3c 2023-09-06T20:16:16,088 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/d4fcb082db5d4f81a84266962e575f3c, entries=150, sequenceid=453, filesize=12.0 K 2023-09-06T20:16:16,088 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.HRegion(3022): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 7062ed4ddcd76aee2adae17e574b6875 in 874ms, sequenceid=453, compaction requested=true 2023-09-06T20:16:16,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:16,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:16,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=197}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=197 2023-09-06T20:16:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=197 2023-09-06T20:16:16,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=197, resume processing ppid=196 2023-09-06T20:16:16,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=197, ppid=196, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0280 sec 2023-09-06T20:16:16,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=196, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=196, table=TestAcidGuarantees in 1.0320 sec 2023-09-06T20:16:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=196 2023-09-06T20:16:16,164 INFO [Thread-3234 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 196 completed 2023-09-06T20:16:16,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:16:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=198, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=198, table=TestAcidGuarantees 2023-09-06T20:16:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=198 2023-09-06T20:16:16,168 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=198, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=198, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:16:16,168 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=198, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=198, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:16:16,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=199, ppid=198, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:16:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=198 2023-09-06T20:16:16,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=199 2023-09-06T20:16:16,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:16,322 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2023-09-06T20:16:16,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:16,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:16,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:16,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:16,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:16,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:16,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309067b98905d125b4b8f81a3afe1a9fa39da_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031375338/Put/seqid=0 2023-09-06T20:16:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:16,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:16,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031436456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:16,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031436457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:16,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031436458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:16,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031436459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:16,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031436460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=198 2023-09-06T20:16:16,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031436560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:16,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031436561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:16,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031436563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:16,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031436563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:16,739 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309067b98905d125b4b8f81a3afe1a9fa39da_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309067b98905d125b4b8f81a3afe1a9fa39da_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:16,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/beeba11753084194bcb3e2a5d0469256, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:16,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/beeba11753084194bcb3e2a5d0469256 is 175, key is test_row_0/A:col10/1694031375338/Put/seqid=0 2023-09-06T20:16:16,747 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=480, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/beeba11753084194bcb3e2a5d0469256 2023-09-06T20:16:16,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/10621d5d6c9148499779c10711e8251b is 50, key is test_row_0/B:col10/1694031375338/Put/seqid=0 2023-09-06T20:16:16,756 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/10621d5d6c9148499779c10711e8251b 2023-09-06T20:16:16,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/b9173565dd904453ae1c34a771ec8108 is 50, key is test_row_0/C:col10/1694031375338/Put/seqid=0 2023-09-06T20:16:16,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:16,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031436762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:16,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031436764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:16,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031436766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:16,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:16,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031436767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:16,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=198 2023-09-06T20:16:16,789 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/b9173565dd904453ae1c34a771ec8108 2023-09-06T20:16:16,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/beeba11753084194bcb3e2a5d0469256 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/beeba11753084194bcb3e2a5d0469256 2023-09-06T20:16:16,797 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/beeba11753084194bcb3e2a5d0469256, entries=150, sequenceid=480, filesize=30.5 K 2023-09-06T20:16:16,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/10621d5d6c9148499779c10711e8251b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/10621d5d6c9148499779c10711e8251b 2023-09-06T20:16:16,803 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/10621d5d6c9148499779c10711e8251b, entries=150, sequenceid=480, filesize=12.0 K 2023-09-06T20:16:16,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/b9173565dd904453ae1c34a771ec8108 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/b9173565dd904453ae1c34a771ec8108 2023-09-06T20:16:16,808 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/b9173565dd904453ae1c34a771ec8108, entries=150, sequenceid=480, filesize=12.0 K 2023-09-06T20:16:16,809 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.HRegion(3022): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 7062ed4ddcd76aee2adae17e574b6875 in 487ms, sequenceid=480, compaction requested=true 2023-09-06T20:16:16,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:16,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:16,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=199}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=199 2023-09-06T20:16:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=199 2023-09-06T20:16:16,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=199, resume processing ppid=198 2023-09-06T20:16:16,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=199, ppid=198, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 642 msec 2023-09-06T20:16:16,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=198, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=198, table=TestAcidGuarantees in 647 msec 2023-09-06T20:16:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:17,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-06T20:16:17,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:17,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:17,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:17,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:17,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:17,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:17,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906814e523b04dc4d5fb9888a0fcadb5b42_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031376458/Put/seqid=0 2023-09-06T20:16:17,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031437094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031437097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031437099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031437099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031437200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031437202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031437204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031437204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=198 2023-09-06T20:16:17,271 INFO [Thread-3234 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 198 completed 2023-09-06T20:16:17,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-06T20:16:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=200, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=200, table=TestAcidGuarantees 2023-09-06T20:16:17,274 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=200, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=200, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-06T20:16:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=200 2023-09-06T20:16:17,275 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=200, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=200, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-06T20:16:17,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=201, ppid=200, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-06T20:16:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=200 2023-09-06T20:16:17,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031437404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031437407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031437407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031437407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=201 2023-09-06T20:16:17,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:17,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:17,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:17,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=201}] handler.RSProcedureHandler(53): pid=201 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:17,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=201 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=201 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:17,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45568 deadline: 1694031437470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,487 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906814e523b04dc4d5fb9888a0fcadb5b42_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906814e523b04dc4d5fb9888a0fcadb5b42_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:17,488 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/181606fa8d034c8d8f27f1bac5123d0d, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:17,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/181606fa8d034c8d8f27f1bac5123d0d is 175, key is test_row_0/A:col10/1694031376458/Put/seqid=0 2023-09-06T20:16:17,496 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=491, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/181606fa8d034c8d8f27f1bac5123d0d 2023-09-06T20:16:17,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/b11105dfcfcb40959db3d699f67a6e84 is 50, key is test_row_0/B:col10/1694031376458/Put/seqid=0 2023-09-06T20:16:17,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/b11105dfcfcb40959db3d699f67a6e84 2023-09-06T20:16:17,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/4864d2014bd74f79942410fbfe43078d is 50, key is test_row_0/C:col10/1694031376458/Put/seqid=0 2023-09-06T20:16:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=200 2023-09-06T20:16:17,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=201 2023-09-06T20:16:17,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:17,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:17,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:17,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] handler.RSProcedureHandler(53): pid=201 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:17,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=201 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:17,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=201 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:17,647 DEBUG [Thread-3237 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0d3acf9e to 127.0.0.1:58481 2023-09-06T20:16:17,647 DEBUG [Thread-3237 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:17,648 DEBUG [Thread-3241 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7c4a00fc to 127.0.0.1:58481 2023-09-06T20:16:17,648 DEBUG [Thread-3241 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:17,648 DEBUG [Thread-3239 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2876f366 to 127.0.0.1:58481 2023-09-06T20:16:17,648 DEBUG [Thread-3239 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:17,650 DEBUG [Thread-3235 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x62e4708a to 127.0.0.1:58481 2023-09-06T20:16:17,650 DEBUG [Thread-3235 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:17,652 DEBUG [Thread-3243 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0c17d3e8 to 127.0.0.1:58481 2023-09-06T20:16:17,652 DEBUG [Thread-3243 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:17,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45630 deadline: 1694031437707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45572 deadline: 1694031437710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45588 deadline: 1694031437712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:16:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:45642 deadline: 1694031437713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:17,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=201 2023-09-06T20:16:17,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:17,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:17,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:17,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=201}] handler.RSProcedureHandler(53): pid=201 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:17,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=201 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=201 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=200 2023-09-06T20:16:17,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=201 2023-09-06T20:16:17,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:17,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:17,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:17,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=201}] handler.RSProcedureHandler(53): pid=201 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:17,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=201 java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(4003): Remote procedure failed, pid=201 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:16:17,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/4864d2014bd74f79942410fbfe43078d 2023-09-06T20:16:17,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/181606fa8d034c8d8f27f1bac5123d0d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/181606fa8d034c8d8f27f1bac5123d0d 2023-09-06T20:16:17,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/181606fa8d034c8d8f27f1bac5123d0d, entries=200, sequenceid=491, filesize=39.0 K 2023-09-06T20:16:17,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/b11105dfcfcb40959db3d699f67a6e84 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b11105dfcfcb40959db3d699f67a6e84 2023-09-06T20:16:17,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b11105dfcfcb40959db3d699f67a6e84, entries=150, sequenceid=491, filesize=12.0 K 2023-09-06T20:16:17,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/4864d2014bd74f79942410fbfe43078d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/4864d2014bd74f79942410fbfe43078d 2023-09-06T20:16:17,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/4864d2014bd74f79942410fbfe43078d, entries=150, sequenceid=491, filesize=12.0 K 2023-09-06T20:16:17,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7062ed4ddcd76aee2adae17e574b6875 in 876ms, sequenceid=491, compaction requested=true 2023-09-06T20:16:17,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:17,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:A, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:16:17,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:17,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:B, priority=-2147483648, current under compaction store size is 2 2023-09-06T20:16:17,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:17,944 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2023-09-06T20:16:17,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7062ed4ddcd76aee2adae17e574b6875:C, priority=-2147483648, current under compaction store size is 3 2023-09-06T20:16:17,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:17,944 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2023-09-06T20:16:17,946 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 165838 starting at candidate #0 after considering 6 permutations with 6 in ratio 2023-09-06T20:16:17,946 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/A is initiating minor compaction (all files) 2023-09-06T20:16:17,946 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62374 starting at candidate #0 after considering 6 permutations with 6 in ratio 2023-09-06T20:16:17,946 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/B is initiating minor compaction (all files) 2023-09-06T20:16:17,946 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/A in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:17,946 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/B in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:17,946 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/2c7a0a68a96f4960a4135783b17b5c8c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6ac777260bc848bc87085886ec34e748, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6af8a31a976d46308d3bb43ee0e6b589, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/beeba11753084194bcb3e2a5d0469256, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/181606fa8d034c8d8f27f1bac5123d0d] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=162.0 K 2023-09-06T20:16:17,946 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/2846ab78a0c741e289658017bd621a17, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d6bee103de644ecca98c5015e87815cf, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/cbc851d8edc1489682f849ba55e6175b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/10621d5d6c9148499779c10711e8251b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b11105dfcfcb40959db3d699f67a6e84] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=60.9 K 2023-09-06T20:16:17,946 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:17,946 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. files: [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/2c7a0a68a96f4960a4135783b17b5c8c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6ac777260bc848bc87085886ec34e748, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6af8a31a976d46308d3bb43ee0e6b589, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/beeba11753084194bcb3e2a5d0469256, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/181606fa8d034c8d8f27f1bac5123d0d] 2023-09-06T20:16:17,946 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 2846ab78a0c741e289658017bd621a17, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1694031373047 2023-09-06T20:16:17,946 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2c7a0a68a96f4960a4135783b17b5c8c, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1694031373047 2023-09-06T20:16:17,946 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting d6bee103de644ecca98c5015e87815cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1694031373067 2023-09-06T20:16:17,946 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6ac777260bc848bc87085886ec34e748, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1694031373067 2023-09-06T20:16:17,947 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting cbc851d8edc1489682f849ba55e6175b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1694031374191 2023-09-06T20:16:17,947 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6af8a31a976d46308d3bb43ee0e6b589, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1694031374191 2023-09-06T20:16:17,947 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 10621d5d6c9148499779c10711e8251b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1694031375335 2023-09-06T20:16:17,947 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting beeba11753084194bcb3e2a5d0469256, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1694031375335 2023-09-06T20:16:17,947 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b11105dfcfcb40959db3d699f67a6e84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1694031376456 2023-09-06T20:16:17,947 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] compactions.Compactor(223): Compacting 181606fa8d034c8d8f27f1bac5123d0d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1694031376451 2023-09-06T20:16:17,956 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#B#compaction#633 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:17,956 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/9e0f634553d34cf19cb7737b0d9b947a is 50, key is test_row_0/B:col10/1694031376458/Put/seqid=0 2023-09-06T20:16:17,962 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:17,963 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230906779441497c32416580c5596c0e9f3e8c_7062ed4ddcd76aee2adae17e574b6875 store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:17,967 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230906779441497c32416580c5596c0e9f3e8c_7062ed4ddcd76aee2adae17e574b6875, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:17,967 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906779441497c32416580c5596c0e9f3e8c_7062ed4ddcd76aee2adae17e574b6875 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:18,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=201 2023-09-06T20:16:18,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:18,038 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-06T20:16:18,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:18,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:18,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:18,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:18,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:18,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:18,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309061fb99a13bc274cde9192c7dd475ac3a0_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031377098/Put/seqid=0 2023-09-06T20:16:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33275 {}] regionserver.HRegion(8537): Flush requested on 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:18,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. as already flushing 2023-09-06T20:16:18,212 DEBUG [Thread-3226 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6090edd2 to 127.0.0.1:58481 2023-09-06T20:16:18,212 DEBUG [Thread-3226 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:18,215 DEBUG [Thread-3228 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0ef38761 to 127.0.0.1:58481 2023-09-06T20:16:18,215 DEBUG [Thread-3228 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:18,217 DEBUG [Thread-3230 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5a125dbe to 127.0.0.1:58481 2023-09-06T20:16:18,217 DEBUG [Thread-3230 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:18,218 DEBUG [Thread-3232 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3839a9cc to 127.0.0.1:58481 2023-09-06T20:16:18,218 DEBUG [Thread-3232 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:18,368 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/9e0f634553d34cf19cb7737b0d9b947a as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9e0f634553d34cf19cb7737b0d9b947a 2023-09-06T20:16:18,371 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#A#compaction#634 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:18,371 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/3706b89e33f948dd891a4aa5b6db2bc1 is 175, key is test_row_0/A:col10/1694031376458/Put/seqid=0 2023-09-06T20:16:18,372 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 5 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/B of 7062ed4ddcd76aee2adae17e574b6875 into 9e0f634553d34cf19cb7737b0d9b947a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:18,372 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:18,372 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/B, priority=11, startTime=1694031377944; duration=0sec 2023-09-06T20:16:18,372 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-06T20:16:18,372 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:B 2023-09-06T20:16:18,372 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2023-09-06T20:16:18,373 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62374 starting at candidate #0 after considering 6 permutations with 6 in ratio 2023-09-06T20:16:18,373 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1534): 7062ed4ddcd76aee2adae17e574b6875/C is initiating minor compaction (all files) 2023-09-06T20:16:18,373 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 7062ed4ddcd76aee2adae17e574b6875/C in TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:18,374 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/e4d7aae9c66849c788e07b60d66826d7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/7dec6b6399dd41bbb2d523b703c50a36, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/d4fcb082db5d4f81a84266962e575f3c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/b9173565dd904453ae1c34a771ec8108, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/4864d2014bd74f79942410fbfe43078d] into tmpdir=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp, totalSize=60.9 K 2023-09-06T20:16:18,374 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting e4d7aae9c66849c788e07b60d66826d7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1694031373047 2023-09-06T20:16:18,374 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 7dec6b6399dd41bbb2d523b703c50a36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1694031373067 2023-09-06T20:16:18,375 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting d4fcb082db5d4f81a84266962e575f3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1694031374191 2023-09-06T20:16:18,375 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting b9173565dd904453ae1c34a771ec8108, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1694031375335 2023-09-06T20:16:18,375 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] compactions.Compactor(223): Compacting 4864d2014bd74f79942410fbfe43078d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1694031376456 2023-09-06T20:16:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=200 2023-09-06T20:16:18,378 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/3706b89e33f948dd891a4aa5b6db2bc1 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/3706b89e33f948dd891a4aa5b6db2bc1 2023-09-06T20:16:18,383 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 5 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/A of 7062ed4ddcd76aee2adae17e574b6875 into 3706b89e33f948dd891a4aa5b6db2bc1(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:18,383 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:18,383 INFO [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/A, priority=11, startTime=1694031377944; duration=0sec 2023-09-06T20:16:18,383 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:18,383 DEBUG [RS:0;jenkins-hbase3:33275-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:A 2023-09-06T20:16:18,384 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7062ed4ddcd76aee2adae17e574b6875#C#compaction#636 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:16:18,384 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/a7d26cc9e57b4c7fa5c702776efcac22 is 50, key is test_row_0/C:col10/1694031376458/Put/seqid=0 2023-09-06T20:16:18,449 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309061fb99a13bc274cde9192c7dd475ac3a0_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309061fb99a13bc274cde9192c7dd475ac3a0_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:18,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/e4935c956ba9472e90a0c88f8f712fb3, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:18,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/e4935c956ba9472e90a0c88f8f712fb3 is 175, key is test_row_0/A:col10/1694031377098/Put/seqid=0 2023-09-06T20:16:18,791 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/a7d26cc9e57b4c7fa5c702776efcac22 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/a7d26cc9e57b4c7fa5c702776efcac22 2023-09-06T20:16:18,795 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 5 (all) file(s) in 7062ed4ddcd76aee2adae17e574b6875/C of 7062ed4ddcd76aee2adae17e574b6875 into a7d26cc9e57b4c7fa5c702776efcac22(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:16:18,795 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:18,795 INFO [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875., storeName=7062ed4ddcd76aee2adae17e574b6875/C, priority=11, startTime=1694031377944; duration=0sec 2023-09-06T20:16:18,795 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:16:18,795 DEBUG [RS:0;jenkins-hbase3:33275-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7062ed4ddcd76aee2adae17e574b6875:C 2023-09-06T20:16:18,854 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=516, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/e4935c956ba9472e90a0c88f8f712fb3 2023-09-06T20:16:18,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/9868d1d651e442238bab0df198d38750 is 50, key is test_row_0/B:col10/1694031377098/Put/seqid=0 2023-09-06T20:16:19,262 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/9868d1d651e442238bab0df198d38750 2023-09-06T20:16:19,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/d73b3cf579a74f18ba74d73502a97185 is 50, key is test_row_0/C:col10/1694031377098/Put/seqid=0 2023-09-06T20:16:19,272 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/d73b3cf579a74f18ba74d73502a97185 2023-09-06T20:16:19,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/e4935c956ba9472e90a0c88f8f712fb3 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/e4935c956ba9472e90a0c88f8f712fb3 2023-09-06T20:16:19,278 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/e4935c956ba9472e90a0c88f8f712fb3, entries=150, sequenceid=516, filesize=30.5 K 2023-09-06T20:16:19,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/9868d1d651e442238bab0df198d38750 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9868d1d651e442238bab0df198d38750 2023-09-06T20:16:19,281 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9868d1d651e442238bab0df198d38750, entries=150, sequenceid=516, filesize=12.0 K 2023-09-06T20:16:19,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/d73b3cf579a74f18ba74d73502a97185 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/d73b3cf579a74f18ba74d73502a97185 2023-09-06T20:16:19,285 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/d73b3cf579a74f18ba74d73502a97185, entries=150, sequenceid=516, filesize=12.0 K 2023-09-06T20:16:19,285 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=26.84 KB/27480 for 7062ed4ddcd76aee2adae17e574b6875 in 1247ms, sequenceid=516, compaction requested=false 2023-09-06T20:16:19,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.HRegion(2520): Flush status journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:19,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:19,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=201}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=201 2023-09-06T20:16:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster(3995): Remote procedure done, pid=201 2023-09-06T20:16:19,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=201, resume processing ppid=200 2023-09-06T20:16:19,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=201, ppid=200, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0110 sec 2023-09-06T20:16:19,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=200, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=200, table=TestAcidGuarantees in 2.0160 sec 2023-09-06T20:16:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=200 2023-09-06T20:16:19,379 INFO [Thread-3234 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 200 completed 2023-09-06T20:16:19,483 DEBUG [Thread-3224 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x28fecf47 to 127.0.0.1:58481 2023-09-06T20:16:19,484 DEBUG [Thread-3224 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:19,484 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2023-09-06T20:16:19,484 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2023-09-06T20:16:19,484 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2023-09-06T20:16:19,484 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2023-09-06T20:16:19,484 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2023-09-06T20:16:19,484 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2023-09-06T20:16:19,484 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2023-09-06T20:16:19,484 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(398): read 5851 2023-09-06T20:16:19,484 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(398): read 5805 2023-09-06T20:16:19,484 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(398): read 6031 2023-09-06T20:16:19,484 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(398): read 5864 2023-09-06T20:16:19,484 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(398): read 5853 2023-09-06T20:16:19,484 INFO [Listener at localhost/35961 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2023-09-06T20:16:19,484 INFO [Listener at localhost/35961 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:16:19,484 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x18fcc509 to 127.0.0.1:58481 2023-09-06T20:16:19,484 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:19,485 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2023-09-06T20:16:19,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable TestAcidGuarantees 2023-09-06T20:16:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=202, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2023-09-06T20:16:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=202 2023-09-06T20:16:19,488 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031379488"}]},"ts":"1694031379488"} 2023-09-06T20:16:19,489 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2023-09-06T20:16:19,490 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2023-09-06T20:16:19,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=203, ppid=202, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=7062ed4ddcd76aee2adae17e574b6875, UNASSIGN}] 2023-09-06T20:16:19,492 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=203, ppid=202, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=7062ed4ddcd76aee2adae17e574b6875, UNASSIGN 2023-09-06T20:16:19,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=203 updating hbase:meta row=7062ed4ddcd76aee2adae17e574b6875, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:19,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=204, ppid=203, state=RUNNABLE; CloseRegionProcedure 7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268}] 2023-09-06T20:16:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=202 2023-09-06T20:16:19,645 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] handler.UnassignRegionHandler(119): Close 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:19,645 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HRegion(1668): Closing 7062ed4ddcd76aee2adae17e574b6875, disabling compactions & flushes 2023-09-06T20:16:19,645 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:19,645 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:19,645 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. after waiting 0 ms 2023-09-06T20:16:19,645 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:19,646 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HRegion(2819): Flushing 7062ed4ddcd76aee2adae17e574b6875 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2023-09-06T20:16:19,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=A 2023-09-06T20:16:19,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:19,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=B 2023-09-06T20:16:19,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:19,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 7062ed4ddcd76aee2adae17e574b6875, store=C 2023-09-06T20:16:19,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-06T20:16:19,649 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906cf7371bd2f9b4226b264e763efbf5da4_7062ed4ddcd76aee2adae17e574b6875 is 50, key is test_row_0/A:col10/1694031379482/Put/seqid=0 2023-09-06T20:16:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=202 2023-09-06T20:16:20,056 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230906cf7371bd2f9b4226b264e763efbf5da4_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906cf7371bd2f9b4226b264e763efbf5da4_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:20,057 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/3816b6d86c4942d9a0d9499bc9b9abba, store: [table=TestAcidGuarantees family=A region=7062ed4ddcd76aee2adae17e574b6875] 2023-09-06T20:16:20,058 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/3816b6d86c4942d9a0d9499bc9b9abba is 175, key is test_row_0/A:col10/1694031379482/Put/seqid=0 2023-09-06T20:16:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=202 2023-09-06T20:16:20,461 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=527, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/3816b6d86c4942d9a0d9499bc9b9abba 2023-09-06T20:16:20,466 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/7a750a4068884a559462596b84367a3d is 50, key is test_row_0/B:col10/1694031379482/Put/seqid=0 2023-09-06T20:16:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=202 2023-09-06T20:16:20,869 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/7a750a4068884a559462596b84367a3d 2023-09-06T20:16:20,874 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/49ceffdaf5d2417baebe2a4172f3e451 is 50, key is test_row_0/C:col10/1694031379482/Put/seqid=0 2023-09-06T20:16:21,278 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/49ceffdaf5d2417baebe2a4172f3e451 2023-09-06T20:16:21,281 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/A/3816b6d86c4942d9a0d9499bc9b9abba as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/3816b6d86c4942d9a0d9499bc9b9abba 2023-09-06T20:16:21,284 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/3816b6d86c4942d9a0d9499bc9b9abba, entries=100, sequenceid=527, filesize=22.0 K 2023-09-06T20:16:21,285 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/B/7a750a4068884a559462596b84367a3d as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7a750a4068884a559462596b84367a3d 2023-09-06T20:16:21,291 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7a750a4068884a559462596b84367a3d, entries=100, sequenceid=527, filesize=9.6 K 2023-09-06T20:16:21,291 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/.tmp/C/49ceffdaf5d2417baebe2a4172f3e451 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/49ceffdaf5d2417baebe2a4172f3e451 2023-09-06T20:16:21,294 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/49ceffdaf5d2417baebe2a4172f3e451, entries=100, sequenceid=527, filesize=9.6 K 2023-09-06T20:16:21,295 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HRegion(3022): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 7062ed4ddcd76aee2adae17e574b6875 in 1650ms, sequenceid=527, compaction requested=true 2023-09-06T20:16:21,303 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/a2f9cf3c38cf437fb2cb11622f09b132, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/11ff455875ef41da86eef1d6b58fb0ba, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/afd809d30e294794961b123369f6cded, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/1d5cb59d446c4aee9ce847d505638c02, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/ed6d6a8bbca74397b5a741d1b9d48a01, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/da25c95ebb154347992dc98fef2fae2b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/ebad05b8587642ec8d7e8450e02a0d7b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/24cef1903e0c4ede91f9533e0fe940a1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c6d639737f5343cf87f856a46827091c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f89270012a1d40bea84f3d6cf4c58435, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/afebe6ba93dd4cdc961dd670a02768b4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/125b5830883c43e2bbff736194044fa8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c9444d58b5a54fd28bccb336a34ff694, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/17f16c4ca58840d1876f580184928d80, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6510671bd43748e3a4025635073f641d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/9d0b4aac07754a568a79a55f7bc90178, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6d4ad27dfb9e4e7c8991499b1ffa2fd2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/b9f54d0cb82a416bbb6cd88aa780b5e0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c9fa613c3f5d45989d1d149e20598092, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/b592b8aa9daa4055b87d4ad678c9026b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/004d84b6791746f28f6fc69badb75bcb, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/eca4b362f1144c47b69f5f738d4b2a1e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/a8734cb9e3c444778e6b70d8653ec991, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/83d07452fdb44137877de30b6aae1033, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/fbabfe6c9ec34ec78bcb8bb914f0918f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f4075035286b48fc9fa225ca5bedf556, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/fa4087b6f06a4f8b87931a6cf3fe94d8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f9c224211f444052998b8ea42785412f, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/2c7a0a68a96f4960a4135783b17b5c8c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6ac777260bc848bc87085886ec34e748, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6af8a31a976d46308d3bb43ee0e6b589, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/beeba11753084194bcb3e2a5d0469256, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/181606fa8d034c8d8f27f1bac5123d0d] to archive 2023-09-06T20:16:21,303 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:16:21,305 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/a2f9cf3c38cf437fb2cb11622f09b132 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/a2f9cf3c38cf437fb2cb11622f09b132 2023-09-06T20:16:21,306 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/11ff455875ef41da86eef1d6b58fb0ba to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/11ff455875ef41da86eef1d6b58fb0ba 2023-09-06T20:16:21,307 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/afd809d30e294794961b123369f6cded to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/afd809d30e294794961b123369f6cded 2023-09-06T20:16:21,308 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/1d5cb59d446c4aee9ce847d505638c02 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/1d5cb59d446c4aee9ce847d505638c02 2023-09-06T20:16:21,309 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/ed6d6a8bbca74397b5a741d1b9d48a01 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/ed6d6a8bbca74397b5a741d1b9d48a01 2023-09-06T20:16:21,310 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/da25c95ebb154347992dc98fef2fae2b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/da25c95ebb154347992dc98fef2fae2b 2023-09-06T20:16:21,311 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/ebad05b8587642ec8d7e8450e02a0d7b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/ebad05b8587642ec8d7e8450e02a0d7b 2023-09-06T20:16:21,312 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/24cef1903e0c4ede91f9533e0fe940a1 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/24cef1903e0c4ede91f9533e0fe940a1 2023-09-06T20:16:21,313 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c6d639737f5343cf87f856a46827091c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c6d639737f5343cf87f856a46827091c 2023-09-06T20:16:21,314 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f89270012a1d40bea84f3d6cf4c58435 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f89270012a1d40bea84f3d6cf4c58435 2023-09-06T20:16:21,315 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/afebe6ba93dd4cdc961dd670a02768b4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/afebe6ba93dd4cdc961dd670a02768b4 2023-09-06T20:16:21,316 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/125b5830883c43e2bbff736194044fa8 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/125b5830883c43e2bbff736194044fa8 2023-09-06T20:16:21,317 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c9444d58b5a54fd28bccb336a34ff694 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c9444d58b5a54fd28bccb336a34ff694 2023-09-06T20:16:21,318 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/17f16c4ca58840d1876f580184928d80 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/17f16c4ca58840d1876f580184928d80 2023-09-06T20:16:21,319 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6510671bd43748e3a4025635073f641d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6510671bd43748e3a4025635073f641d 2023-09-06T20:16:21,320 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/9d0b4aac07754a568a79a55f7bc90178 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/9d0b4aac07754a568a79a55f7bc90178 2023-09-06T20:16:21,320 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6d4ad27dfb9e4e7c8991499b1ffa2fd2 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6d4ad27dfb9e4e7c8991499b1ffa2fd2 2023-09-06T20:16:21,321 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/b9f54d0cb82a416bbb6cd88aa780b5e0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/b9f54d0cb82a416bbb6cd88aa780b5e0 2023-09-06T20:16:21,322 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c9fa613c3f5d45989d1d149e20598092 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/c9fa613c3f5d45989d1d149e20598092 2023-09-06T20:16:21,323 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/b592b8aa9daa4055b87d4ad678c9026b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/b592b8aa9daa4055b87d4ad678c9026b 2023-09-06T20:16:21,325 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/004d84b6791746f28f6fc69badb75bcb to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/004d84b6791746f28f6fc69badb75bcb 2023-09-06T20:16:21,326 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/eca4b362f1144c47b69f5f738d4b2a1e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/eca4b362f1144c47b69f5f738d4b2a1e 2023-09-06T20:16:21,327 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/a8734cb9e3c444778e6b70d8653ec991 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/a8734cb9e3c444778e6b70d8653ec991 2023-09-06T20:16:21,327 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/83d07452fdb44137877de30b6aae1033 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/83d07452fdb44137877de30b6aae1033 2023-09-06T20:16:21,328 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/fbabfe6c9ec34ec78bcb8bb914f0918f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/fbabfe6c9ec34ec78bcb8bb914f0918f 2023-09-06T20:16:21,329 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f4075035286b48fc9fa225ca5bedf556 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f4075035286b48fc9fa225ca5bedf556 2023-09-06T20:16:21,330 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/fa4087b6f06a4f8b87931a6cf3fe94d8 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/fa4087b6f06a4f8b87931a6cf3fe94d8 2023-09-06T20:16:21,331 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f9c224211f444052998b8ea42785412f to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/f9c224211f444052998b8ea42785412f 2023-09-06T20:16:21,332 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/2c7a0a68a96f4960a4135783b17b5c8c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/2c7a0a68a96f4960a4135783b17b5c8c 2023-09-06T20:16:21,333 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6ac777260bc848bc87085886ec34e748 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6ac777260bc848bc87085886ec34e748 2023-09-06T20:16:21,334 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6af8a31a976d46308d3bb43ee0e6b589 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/6af8a31a976d46308d3bb43ee0e6b589 2023-09-06T20:16:21,334 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/beeba11753084194bcb3e2a5d0469256 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/beeba11753084194bcb3e2a5d0469256 2023-09-06T20:16:21,336 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/181606fa8d034c8d8f27f1bac5123d0d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/181606fa8d034c8d8f27f1bac5123d0d 2023-09-06T20:16:21,339 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d54d79254cc14740ad84358d6cae82f6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b09c5c4349e84c03a031f8be3f13b3f2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7bb3afd7b4944d5fb4a3119e72133a03, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9618e7d10f96414087781b95a8bb6ca3, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/270695a6ac344b318ea54e7b4e90dad9, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9fb120009cce4e418af659a1e01c84c0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/ba2a75fbc530463fa6fcc4d8dd629c4c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/e7197645ad1c4b7d9a72212a823ee0a0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/09ffe86814b94b0f8523dbf9c2b14ff4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/a7bc61f674664d908751ec5fbef1f2bd, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/bdc513f828bd4d0f8718471ebe06e627, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/528408b3007348a6ad5d1af3614304c6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/903fc1b0cbe647fc822c1dacb3dbbc25, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/24657704c57348cd8215bfd0b0a7b723, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/46d3211238ff45ac8aee15eb633a4466, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/4c52b3ba42e04311803cb76245f793b1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/c1d0a505707343a687b29bcb8c7b4fb0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/eb9fc4bdb54d41d8ac2a7dabd1d4e05e, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/5b56ef0094694b6994f8675ad43c4085, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b20a681acce345dc85f5ac0b313b3dca, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/06427bebd89d49bf8cdc698aa003648c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/c722acadb4304e2e9b7c5cd60144ba1a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/33bd998fc65840259cf391b0170522bd, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/4d91d3a619e74fb9ae196f93128c738a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d72225a9231a4a328bb04ace266ec403, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/0586a50467d54f188475c6ec38b9961d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7174c6e9080a493bad26b7098175fec0, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/2846ab78a0c741e289658017bd621a17, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7db4ce67d65646c5afba469e69f01d00, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d6bee103de644ecca98c5015e87815cf, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/cbc851d8edc1489682f849ba55e6175b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/10621d5d6c9148499779c10711e8251b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b11105dfcfcb40959db3d699f67a6e84] to archive 2023-09-06T20:16:21,339 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:16:21,340 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d54d79254cc14740ad84358d6cae82f6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d54d79254cc14740ad84358d6cae82f6 2023-09-06T20:16:21,341 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b09c5c4349e84c03a031f8be3f13b3f2 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b09c5c4349e84c03a031f8be3f13b3f2 2023-09-06T20:16:21,342 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7bb3afd7b4944d5fb4a3119e72133a03 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7bb3afd7b4944d5fb4a3119e72133a03 2023-09-06T20:16:21,343 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9618e7d10f96414087781b95a8bb6ca3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9618e7d10f96414087781b95a8bb6ca3 2023-09-06T20:16:21,344 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/270695a6ac344b318ea54e7b4e90dad9 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/270695a6ac344b318ea54e7b4e90dad9 2023-09-06T20:16:21,345 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9fb120009cce4e418af659a1e01c84c0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9fb120009cce4e418af659a1e01c84c0 2023-09-06T20:16:21,346 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/ba2a75fbc530463fa6fcc4d8dd629c4c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/ba2a75fbc530463fa6fcc4d8dd629c4c 2023-09-06T20:16:21,347 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/e7197645ad1c4b7d9a72212a823ee0a0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/e7197645ad1c4b7d9a72212a823ee0a0 2023-09-06T20:16:21,347 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/09ffe86814b94b0f8523dbf9c2b14ff4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/09ffe86814b94b0f8523dbf9c2b14ff4 2023-09-06T20:16:21,348 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/a7bc61f674664d908751ec5fbef1f2bd to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/a7bc61f674664d908751ec5fbef1f2bd 2023-09-06T20:16:21,349 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/bdc513f828bd4d0f8718471ebe06e627 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/bdc513f828bd4d0f8718471ebe06e627 2023-09-06T20:16:21,350 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/528408b3007348a6ad5d1af3614304c6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/528408b3007348a6ad5d1af3614304c6 2023-09-06T20:16:21,351 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/903fc1b0cbe647fc822c1dacb3dbbc25 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/903fc1b0cbe647fc822c1dacb3dbbc25 2023-09-06T20:16:21,352 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/24657704c57348cd8215bfd0b0a7b723 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/24657704c57348cd8215bfd0b0a7b723 2023-09-06T20:16:21,353 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/46d3211238ff45ac8aee15eb633a4466 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/46d3211238ff45ac8aee15eb633a4466 2023-09-06T20:16:21,354 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/4c52b3ba42e04311803cb76245f793b1 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/4c52b3ba42e04311803cb76245f793b1 2023-09-06T20:16:21,355 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/c1d0a505707343a687b29bcb8c7b4fb0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/c1d0a505707343a687b29bcb8c7b4fb0 2023-09-06T20:16:21,356 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/eb9fc4bdb54d41d8ac2a7dabd1d4e05e to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/eb9fc4bdb54d41d8ac2a7dabd1d4e05e 2023-09-06T20:16:21,357 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/5b56ef0094694b6994f8675ad43c4085 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/5b56ef0094694b6994f8675ad43c4085 2023-09-06T20:16:21,357 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b20a681acce345dc85f5ac0b313b3dca to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b20a681acce345dc85f5ac0b313b3dca 2023-09-06T20:16:21,358 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/06427bebd89d49bf8cdc698aa003648c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/06427bebd89d49bf8cdc698aa003648c 2023-09-06T20:16:21,359 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/c722acadb4304e2e9b7c5cd60144ba1a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/c722acadb4304e2e9b7c5cd60144ba1a 2023-09-06T20:16:21,360 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/33bd998fc65840259cf391b0170522bd to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/33bd998fc65840259cf391b0170522bd 2023-09-06T20:16:21,361 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/4d91d3a619e74fb9ae196f93128c738a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/4d91d3a619e74fb9ae196f93128c738a 2023-09-06T20:16:21,361 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d72225a9231a4a328bb04ace266ec403 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d72225a9231a4a328bb04ace266ec403 2023-09-06T20:16:21,363 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/0586a50467d54f188475c6ec38b9961d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/0586a50467d54f188475c6ec38b9961d 2023-09-06T20:16:21,364 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7174c6e9080a493bad26b7098175fec0 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7174c6e9080a493bad26b7098175fec0 2023-09-06T20:16:21,365 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/2846ab78a0c741e289658017bd621a17 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/2846ab78a0c741e289658017bd621a17 2023-09-06T20:16:21,366 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7db4ce67d65646c5afba469e69f01d00 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7db4ce67d65646c5afba469e69f01d00 2023-09-06T20:16:21,367 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d6bee103de644ecca98c5015e87815cf to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/d6bee103de644ecca98c5015e87815cf 2023-09-06T20:16:21,368 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/cbc851d8edc1489682f849ba55e6175b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/cbc851d8edc1489682f849ba55e6175b 2023-09-06T20:16:21,368 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/10621d5d6c9148499779c10711e8251b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/10621d5d6c9148499779c10711e8251b 2023-09-06T20:16:21,369 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b11105dfcfcb40959db3d699f67a6e84 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/b11105dfcfcb40959db3d699f67a6e84 2023-09-06T20:16:21,372 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/568697960a3b489698903a899f3f2b80, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ec11dc905e6f4aceac00423e3ea857ea, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/0ffb93c767864762a6530392cade6eac, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/5207f13a82914e1588d45deca1413613, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/6ba02f1841014882a53e8b3f9ec29ced, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/1fe21f662f144807b02132fbd280a14d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/152d24a29c9e4b319cc6f8e6f3c413fa, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/176bf07c580f41ca9b3b2b75af92a4d4, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/9271d09d0bec454a8710c5fd815fa3b1, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/9ed31c460ad34a9e8c19201893ec8f7b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/dcbb2c3c534c4df6a358590b6e388a5c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/add8c040ba3c42af8b904b73fd431e37, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/98c64db87ddb4abea2f94bca19e04af6, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/bef224f8b29f4ec8bf11a9169a9349b8, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/46a95227a85e465d9e9d888fe115b969, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/cbeaa654f96f49c18a39b8b04c278a16, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/0f89ddd3f88c4d8bb4948d050a18c962, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/280f55df3c094e46a67e5658dcae0018, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/fc7b3ecc8b4548c5aadaa926364faf1d, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/28acbd6e9b674ecab4150aac215ffe17, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ade4f73560a747e589138040080f4995, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/26c160f39a3d497db0e3adf76900706a, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ff48bb96f3d34e1994d528a2b633954b, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/bf4a067dbafc41e893a572f82d3280b2, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/46cc145beb944b33b9f62975c2e8de77, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/49de05ec2cf9494bbde81ac6d0a26807, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/3931c298b80341b8989afb9437bf5323, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/e4d7aae9c66849c788e07b60d66826d7, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/f74c2135ef8c4f71956a283cbaf2c769, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/7dec6b6399dd41bbb2d523b703c50a36, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/d4fcb082db5d4f81a84266962e575f3c, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/b9173565dd904453ae1c34a771ec8108, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/4864d2014bd74f79942410fbfe43078d] to archive 2023-09-06T20:16:21,372 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:16:21,373 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/568697960a3b489698903a899f3f2b80 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/568697960a3b489698903a899f3f2b80 2023-09-06T20:16:21,374 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ec11dc905e6f4aceac00423e3ea857ea to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ec11dc905e6f4aceac00423e3ea857ea 2023-09-06T20:16:21,375 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/0ffb93c767864762a6530392cade6eac to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/0ffb93c767864762a6530392cade6eac 2023-09-06T20:16:21,376 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/5207f13a82914e1588d45deca1413613 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/5207f13a82914e1588d45deca1413613 2023-09-06T20:16:21,378 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/6ba02f1841014882a53e8b3f9ec29ced to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/6ba02f1841014882a53e8b3f9ec29ced 2023-09-06T20:16:21,378 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/1fe21f662f144807b02132fbd280a14d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/1fe21f662f144807b02132fbd280a14d 2023-09-06T20:16:21,379 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/152d24a29c9e4b319cc6f8e6f3c413fa to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/152d24a29c9e4b319cc6f8e6f3c413fa 2023-09-06T20:16:21,380 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/176bf07c580f41ca9b3b2b75af92a4d4 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/176bf07c580f41ca9b3b2b75af92a4d4 2023-09-06T20:16:21,381 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/9271d09d0bec454a8710c5fd815fa3b1 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/9271d09d0bec454a8710c5fd815fa3b1 2023-09-06T20:16:21,382 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/9ed31c460ad34a9e8c19201893ec8f7b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/9ed31c460ad34a9e8c19201893ec8f7b 2023-09-06T20:16:21,383 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/dcbb2c3c534c4df6a358590b6e388a5c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/dcbb2c3c534c4df6a358590b6e388a5c 2023-09-06T20:16:21,384 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/add8c040ba3c42af8b904b73fd431e37 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/add8c040ba3c42af8b904b73fd431e37 2023-09-06T20:16:21,385 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/98c64db87ddb4abea2f94bca19e04af6 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/98c64db87ddb4abea2f94bca19e04af6 2023-09-06T20:16:21,385 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/bef224f8b29f4ec8bf11a9169a9349b8 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/bef224f8b29f4ec8bf11a9169a9349b8 2023-09-06T20:16:21,386 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/46a95227a85e465d9e9d888fe115b969 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/46a95227a85e465d9e9d888fe115b969 2023-09-06T20:16:21,387 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/cbeaa654f96f49c18a39b8b04c278a16 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/cbeaa654f96f49c18a39b8b04c278a16 2023-09-06T20:16:21,388 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/0f89ddd3f88c4d8bb4948d050a18c962 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/0f89ddd3f88c4d8bb4948d050a18c962 2023-09-06T20:16:21,389 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/280f55df3c094e46a67e5658dcae0018 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/280f55df3c094e46a67e5658dcae0018 2023-09-06T20:16:21,390 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/fc7b3ecc8b4548c5aadaa926364faf1d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/fc7b3ecc8b4548c5aadaa926364faf1d 2023-09-06T20:16:21,391 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/28acbd6e9b674ecab4150aac215ffe17 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/28acbd6e9b674ecab4150aac215ffe17 2023-09-06T20:16:21,392 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ade4f73560a747e589138040080f4995 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ade4f73560a747e589138040080f4995 2023-09-06T20:16:21,393 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/26c160f39a3d497db0e3adf76900706a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/26c160f39a3d497db0e3adf76900706a 2023-09-06T20:16:21,394 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ff48bb96f3d34e1994d528a2b633954b to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/ff48bb96f3d34e1994d528a2b633954b 2023-09-06T20:16:21,395 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/bf4a067dbafc41e893a572f82d3280b2 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/bf4a067dbafc41e893a572f82d3280b2 2023-09-06T20:16:21,395 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/46cc145beb944b33b9f62975c2e8de77 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/46cc145beb944b33b9f62975c2e8de77 2023-09-06T20:16:21,396 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/49de05ec2cf9494bbde81ac6d0a26807 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/49de05ec2cf9494bbde81ac6d0a26807 2023-09-06T20:16:21,397 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/3931c298b80341b8989afb9437bf5323 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/3931c298b80341b8989afb9437bf5323 2023-09-06T20:16:21,398 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/e4d7aae9c66849c788e07b60d66826d7 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/e4d7aae9c66849c788e07b60d66826d7 2023-09-06T20:16:21,398 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/f74c2135ef8c4f71956a283cbaf2c769 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/f74c2135ef8c4f71956a283cbaf2c769 2023-09-06T20:16:21,399 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/7dec6b6399dd41bbb2d523b703c50a36 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/7dec6b6399dd41bbb2d523b703c50a36 2023-09-06T20:16:21,400 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/d4fcb082db5d4f81a84266962e575f3c to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/d4fcb082db5d4f81a84266962e575f3c 2023-09-06T20:16:21,402 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/b9173565dd904453ae1c34a771ec8108 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/b9173565dd904453ae1c34a771ec8108 2023-09-06T20:16:21,403 DEBUG [StoreCloser-TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/4864d2014bd74f79942410fbfe43078d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/4864d2014bd74f79942410fbfe43078d 2023-09-06T20:16:21,406 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/recovered.edits/530.seqid, newMaxSeqId=530, maxSeqId=4 2023-09-06T20:16:21,408 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875. 2023-09-06T20:16:21,408 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] regionserver.HRegion(1622): Region close journal for 7062ed4ddcd76aee2adae17e574b6875: 2023-09-06T20:16:21,410 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=204}] handler.UnassignRegionHandler(163): Closed 7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,410 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=203 updating hbase:meta row=7062ed4ddcd76aee2adae17e574b6875, regionState=CLOSED 2023-09-06T20:16:21,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=204, resume processing ppid=203 2023-09-06T20:16:21,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=204, ppid=203, state=SUCCESS; CloseRegionProcedure 7062ed4ddcd76aee2adae17e574b6875, server=jenkins-hbase3.apache.org,33275,1694031217268 in 1.9170 sec 2023-09-06T20:16:21,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=203, resume processing ppid=202 2023-09-06T20:16:21,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=203, ppid=202, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=7062ed4ddcd76aee2adae17e574b6875, UNASSIGN in 1.9210 sec 2023-09-06T20:16:21,414 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694031381414"}]},"ts":"1694031381414"} 2023-09-06T20:16:21,415 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2023-09-06T20:16:21,417 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2023-09-06T20:16:21,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=202, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.9320 sec 2023-09-06T20:16:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=202 2023-09-06T20:16:21,592 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 202 completed 2023-09-06T20:16:21,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete TestAcidGuarantees 2023-09-06T20:16:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] procedure2.ProcedureExecutor(1032): Stored pid=205, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:16:21,595 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=205, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:16:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=205 2023-09-06T20:16:21,595 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=205, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:16:21,598 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,601 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C, FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/recovered.edits] 2023-09-06T20:16:21,603 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/3706b89e33f948dd891a4aa5b6db2bc1 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/3706b89e33f948dd891a4aa5b6db2bc1 2023-09-06T20:16:21,604 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/3816b6d86c4942d9a0d9499bc9b9abba to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/3816b6d86c4942d9a0d9499bc9b9abba 2023-09-06T20:16:21,605 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/e4935c956ba9472e90a0c88f8f712fb3 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/A/e4935c956ba9472e90a0c88f8f712fb3 2023-09-06T20:16:21,608 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7a750a4068884a559462596b84367a3d to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/7a750a4068884a559462596b84367a3d 2023-09-06T20:16:21,609 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9868d1d651e442238bab0df198d38750 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9868d1d651e442238bab0df198d38750 2023-09-06T20:16:21,610 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9e0f634553d34cf19cb7737b0d9b947a to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/B/9e0f634553d34cf19cb7737b0d9b947a 2023-09-06T20:16:21,612 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/49ceffdaf5d2417baebe2a4172f3e451 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/49ceffdaf5d2417baebe2a4172f3e451 2023-09-06T20:16:21,613 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/a7d26cc9e57b4c7fa5c702776efcac22 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/a7d26cc9e57b4c7fa5c702776efcac22 2023-09-06T20:16:21,614 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/d73b3cf579a74f18ba74d73502a97185 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/C/d73b3cf579a74f18ba74d73502a97185 2023-09-06T20:16:21,616 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/recovered.edits/530.seqid to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875/recovered.edits/530.seqid 2023-09-06T20:16:21,616 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/default/TestAcidGuarantees/7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,617 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2023-09-06T20:16:21,618 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2023-09-06T20:16:21,618 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2023-09-06T20:16:21,621 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090603b8d103ad5d4038a2b74068fd75514d_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090603b8d103ad5d4038a2b74068fd75514d_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,622 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309060881824809c94b24b2de25a2d0a0891c_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309060881824809c94b24b2de25a2d0a0891c_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,623 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090608f823b589f34171919f781e35966017_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090608f823b589f34171919f781e35966017_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,624 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309060d808c7eae214372b72bd067d23ce453_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309060d808c7eae214372b72bd067d23ce453_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,625 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309061fb99a13bc274cde9192c7dd475ac3a0_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309061fb99a13bc274cde9192c7dd475ac3a0_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,626 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906240bf7aa93934ceeb2949d62ff7ca61f_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906240bf7aa93934ceeb2949d62ff7ca61f_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,626 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309062f661341dccd43c8b73c23e6be2ad384_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309062f661341dccd43c8b73c23e6be2ad384_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,627 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906482c6d971e2e43b1a6643c52c44d665b_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906482c6d971e2e43b1a6643c52c44d665b_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,628 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309064ee1f9a299e74366b282fe45c8650373_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309064ee1f9a299e74366b282fe45c8650373_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,629 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309064f85cce75115473587d6cf4343fe1123_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309064f85cce75115473587d6cf4343fe1123_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,630 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090650dc2bd3796c449eabade885147703f0_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090650dc2bd3796c449eabade885147703f0_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,632 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090656e54114f89145b3b1b280ae8dac3d9d_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090656e54114f89145b3b1b280ae8dac3d9d_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,633 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906678856ffc8eb4c1496d63cf7db5f6754_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906678856ffc8eb4c1496d63cf7db5f6754_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,634 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090668af5bd26b8b4f1aa8200f5015e4a29f_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090668af5bd26b8b4f1aa8200f5015e4a29f_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,635 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090672e79e1649a64d88b2730821ffc0bd47_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090672e79e1649a64d88b2730821ffc0bd47_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,636 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309067880cd69f343424abf1a7db0c38b5f69_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309067880cd69f343424abf1a7db0c38b5f69_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,636 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309067b98905d125b4b8f81a3afe1a9fa39da_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309067b98905d125b4b8f81a3afe1a9fa39da_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,637 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090680c580466de04bb4b6b2652dea32eb80_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090680c580466de04bb4b6b2652dea32eb80_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,638 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906814e523b04dc4d5fb9888a0fcadb5b42_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906814e523b04dc4d5fb9888a0fcadb5b42_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,639 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090684724b492faf4cc1a1755f610b1811c2_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023090684724b492faf4cc1a1755f610b1811c2_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,640 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309068b03eee65fac4c49bde0651e9e2d8948_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309068b03eee65fac4c49bde0651e9e2d8948_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,641 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906b8dff0da9a4b4960a201a760b24ff6d5_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906b8dff0da9a4b4960a201a760b24ff6d5_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,642 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c3491e50fc014910be3963550ae34607_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c3491e50fc014910be3963550ae34607_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,643 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c604178c429a476ea01615ca188065f5_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906c604178c429a476ea01615ca188065f5_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,644 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906cf7371bd2f9b4226b264e763efbf5da4_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906cf7371bd2f9b4226b264e763efbf5da4_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,645 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f3cb197da08f47f5ba223d896c8b0a83_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906f3cb197da08f47f5ba223d896c8b0a83_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,646 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906fa691d42d5504784a1709c2a49f154ac_7062ed4ddcd76aee2adae17e574b6875 to hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230906fa691d42d5504784a1709c2a49f154ac_7062ed4ddcd76aee2adae17e574b6875 2023-09-06T20:16:21,647 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2023-09-06T20:16:21,649 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=205, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:16:21,651 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2023-09-06T20:16:21,652 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2023-09-06T20:16:21,653 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=205, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:16:21,653 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2023-09-06T20:16:21,653 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694031381653"}]},"ts":"9223372036854775807"} 2023-09-06T20:16:21,655 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:16:21,655 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 7062ed4ddcd76aee2adae17e574b6875, NAME => 'TestAcidGuarantees,,1694031355551.7062ed4ddcd76aee2adae17e574b6875.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:16:21,655 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2023-09-06T20:16:21,655 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694031381655"}]},"ts":"9223372036854775807"} 2023-09-06T20:16:21,656 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2023-09-06T20:16:21,658 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=205, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-06T20:16:21,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=205, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2023-09-06T20:16:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35153 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=205 2023-09-06T20:16:21,696 INFO [Listener at localhost/35961 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 205 completed 2023-09-06T20:16:21,708 INFO [Listener at localhost/35961 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testMobGetAtomicity Thread=223 (was 222) - Thread LEAK? -, OpenFileDescriptor=583 (was 625), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=514 (was 560), ProcessCount=166 (was 167), AvailableMemoryMB=4839 (was 4861) 2023-09-06T20:16:21,708 INFO [Listener at localhost/35961 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-09-06T20:16:21,709 INFO [Listener at localhost/35961 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:16:21,709 DEBUG [Listener at localhost/35961 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x203b04e8 to 127.0.0.1:58481 2023-09-06T20:16:21,709 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:21,709 DEBUG [Listener at localhost/35961 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-09-06T20:16:21,709 DEBUG [Listener at localhost/35961 {}] util.JVMClusterUtil(257): Found active master hash=79852567, stopped=false 2023-09-06T20:16:21,710 INFO [Listener at localhost/35961 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,35153,1694031215758 2023-09-06T20:16:21,712 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2023-09-06T20:16:21,712 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2023-09-06T20:16:21,712 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:16:21,712 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:16:21,712 INFO [Listener at localhost/35961 {}] procedure2.ProcedureExecutor(630): Stopping 2023-09-06T20:16:21,712 DEBUG [Listener at localhost/35961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:21,712 INFO [Listener at localhost/35961 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,33275,1694031217268' ***** 2023-09-06T20:16:21,712 INFO [Listener at localhost/35961 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-09-06T20:16:21,712 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-09-06T20:16:21,712 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-09-06T20:16:21,712 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.HeapMemoryManager(220): Stopping 2023-09-06T20:16:21,713 INFO [RS:0;jenkins-hbase3:33275 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-09-06T20:16:21,713 INFO [RS:0;jenkins-hbase3:33275 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-09-06T20:16:21,713 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-09-06T20:16:21,713 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(3528): Received CLOSE for 46c263e9fd401abca49830e3d9e96c8e 2023-09-06T20:16:21,713 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:21,713 DEBUG [RS:0;jenkins-hbase3:33275 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:21,713 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-09-06T20:16:21,713 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-09-06T20:16:21,713 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-09-06T20:16:21,713 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-09-06T20:16:21,714 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-09-06T20:16:21,714 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 46c263e9fd401abca49830e3d9e96c8e, disabling compactions & flushes 2023-09-06T20:16:21,714 DEBUG [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(1583): Online Regions={1588230740=hbase:meta,,1.1588230740, 46c263e9fd401abca49830e3d9e96c8e=hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e.} 2023-09-06T20:16:21,714 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e. 2023-09-06T20:16:21,714 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e. 2023-09-06T20:16:21,714 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e. after waiting 0 ms 2023-09-06T20:16:21,714 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e. 2023-09-06T20:16:21,714 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 46c263e9fd401abca49830e3d9e96c8e 1/1 column families, dataSize=78 B heapSize=488 B 2023-09-06T20:16:21,715 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-06T20:16:21,715 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-06T20:16:21,715 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-06T20:16:21,715 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-06T20:16:21,715 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-06T20:16:21,715 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=20.92 KB heapSize=36.24 KB 2023-09-06T20:16:21,716 DEBUG [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 46c263e9fd401abca49830e3d9e96c8e 2023-09-06T20:16:21,730 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/namespace/46c263e9fd401abca49830e3d9e96c8e/.tmp/info/aaeec47d718e4a8396473b21fffe0815 is 45, key is default/info:d/1694031221829/Put/seqid=0 2023-09-06T20:16:21,734 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/.tmp/info/a5732542bce64d27980ce45e85628039 is 143, key is hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e./info:regioninfo/1694031221653/Put/seqid=0 2023-09-06T20:16:21,736 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/namespace/46c263e9fd401abca49830e3d9e96c8e/.tmp/info/aaeec47d718e4a8396473b21fffe0815 2023-09-06T20:16:21,746 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/namespace/46c263e9fd401abca49830e3d9e96c8e/.tmp/info/aaeec47d718e4a8396473b21fffe0815 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/namespace/46c263e9fd401abca49830e3d9e96c8e/info/aaeec47d718e4a8396473b21fffe0815 2023-09-06T20:16:21,747 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-09-06T20:16:21,750 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/namespace/46c263e9fd401abca49830e3d9e96c8e/info/aaeec47d718e4a8396473b21fffe0815, entries=2, sequenceid=6, filesize=4.9 K 2023-09-06T20:16:21,751 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 46c263e9fd401abca49830e3d9e96c8e in 37ms, sequenceid=6, compaction requested=false 2023-09-06T20:16:21,760 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/namespace/46c263e9fd401abca49830e3d9e96c8e/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-09-06T20:16:21,762 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e. 2023-09-06T20:16:21,762 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 46c263e9fd401abca49830e3d9e96c8e: 2023-09-06T20:16:21,762 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1694031221141.46c263e9fd401abca49830e3d9e96c8e. 2023-09-06T20:16:21,917 DEBUG [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(1609): Waiting on 1588230740 2023-09-06T20:16:22,117 DEBUG [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(1609): Waiting on 1588230740 2023-09-06T20:16:22,139 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.26 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/.tmp/info/a5732542bce64d27980ce45e85628039 2023-09-06T20:16:22,150 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-09-06T20:16:22,150 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-09-06T20:16:22,155 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/.tmp/rep_barrier/83aab65bf9564a00b5889918bcffdede is 102, key is TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5./rep_barrier:/1694031246696/DeleteFamily/seqid=0 2023-09-06T20:16:22,158 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/.tmp/rep_barrier/83aab65bf9564a00b5889918bcffdede 2023-09-06T20:16:22,176 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/.tmp/table/9b5b63049e704e9abafff8a1842ef1f0 is 96, key is TestAcidGuarantees,,1694031222089.a659f678dfa70a1da8b63a91043e10d5./table:/1694031246696/DeleteFamily/seqid=0 2023-09-06T20:16:22,179 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.08 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/.tmp/table/9b5b63049e704e9abafff8a1842ef1f0 2023-09-06T20:16:22,183 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/.tmp/info/a5732542bce64d27980ce45e85628039 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/info/a5732542bce64d27980ce45e85628039 2023-09-06T20:16:22,186 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/info/a5732542bce64d27980ce45e85628039, entries=22, sequenceid=93, filesize=7.6 K 2023-09-06T20:16:22,187 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/.tmp/rep_barrier/83aab65bf9564a00b5889918bcffdede as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/rep_barrier/83aab65bf9564a00b5889918bcffdede 2023-09-06T20:16:22,190 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/rep_barrier/83aab65bf9564a00b5889918bcffdede, entries=6, sequenceid=93, filesize=5.9 K 2023-09-06T20:16:22,190 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/.tmp/table/9b5b63049e704e9abafff8a1842ef1f0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/table/9b5b63049e704e9abafff8a1842ef1f0 2023-09-06T20:16:22,193 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/table/9b5b63049e704e9abafff8a1842ef1f0, entries=9, sequenceid=93, filesize=5.8 K 2023-09-06T20:16:22,193 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~20.92 KB/21417, heapSize ~36.20 KB/37064, currentSize=0 B/0 for 1588230740 in 478ms, sequenceid=93, compaction requested=false 2023-09-06T20:16:22,197 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/data/hbase/meta/1588230740/recovered.edits/96.seqid, newMaxSeqId=96, maxSeqId=1 2023-09-06T20:16:22,198 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:16:22,199 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-06T20:16:22,199 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-06T20:16:22,199 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-09-06T20:16:22,317 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,33275,1694031217268; all regions closed. 2023-09-06T20:16:22,325 DEBUG [RS:0;jenkins-hbase3:33275 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/oldWALs 2023-09-06T20:16:22,325 INFO [RS:0;jenkins-hbase3:33275 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C33275%2C1694031217268.meta:.meta(num 1694031220800) 2023-09-06T20:16:22,329 DEBUG [RS:0;jenkins-hbase3:33275 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/oldWALs 2023-09-06T20:16:22,329 INFO [RS:0;jenkins-hbase3:33275 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C33275%2C1694031217268:(num 1694031220327) 2023-09-06T20:16:22,329 DEBUG [RS:0;jenkins-hbase3:33275 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:22,329 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.LeaseManager(133): Closed leases 2023-09-06T20:16:22,329 INFO [RS:0;jenkins-hbase3:33275 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-09-06T20:16:22,329 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-06T20:16:22,330 INFO [RS:0;jenkins-hbase3:33275 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:33275 2023-09-06T20:16:22,336 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2023-09-06T20:16:22,336 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/jenkins-hbase3.apache.org,33275,1694031217268 2023-09-06T20:16:22,339 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,33275,1694031217268] 2023-09-06T20:16:22,339 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,33275,1694031217268; numProcessing=1 2023-09-06T20:16:22,340 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/draining/jenkins-hbase3.apache.org,33275,1694031217268 already deleted, retry=false 2023-09-06T20:16:22,340 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,33275,1694031217268 expired; onlineServers=0 2023-09-06T20:16:22,340 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,35153,1694031215758' ***** 2023-09-06T20:16:22,340 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-09-06T20:16:22,340 DEBUG [M:0;jenkins-hbase3:35153 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2c3a77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-06T20:16:22,341 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,35153,1694031215758 2023-09-06T20:16:22,341 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,35153,1694031215758; all regions closed. 2023-09-06T20:16:22,341 DEBUG [M:0;jenkins-hbase3:35153 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:16:22,341 DEBUG [M:0;jenkins-hbase3:35153 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-09-06T20:16:22,341 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-09-06T20:16:22,341 DEBUG [M:0;jenkins-hbase3:35153 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-09-06T20:16:22,341 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694031219909 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694031219909,5,FailOnTimeoutGroup] 2023-09-06T20:16:22,341 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694031219909 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694031219909,5,FailOnTimeoutGroup] 2023-09-06T20:16:22,341 INFO [M:0;jenkins-hbase3:35153 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-09-06T20:16:22,341 DEBUG [M:0;jenkins-hbase3:35153 {}] master.HMaster(1688): Stopping service threads 2023-09-06T20:16:22,342 INFO [M:0;jenkins-hbase3:35153 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-09-06T20:16:22,342 ERROR [M:0;jenkins-hbase3:35153 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1097616044) connection to localhost/127.0.0.1:42081 from jenkins,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2023-09-06T20:16:22,342 INFO [M:0;jenkins-hbase3:35153 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-09-06T20:16:22,342 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-09-06T20:16:22,342 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2023-09-06T20:16:22,343 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-06T20:16:22,343 DEBUG [M:0;jenkins-hbase3:35153 {}] zookeeper.ZKUtil(347): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2023-09-06T20:16:22,343 WARN [M:0;jenkins-hbase3:35153 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-09-06T20:16:22,343 INFO [M:0;jenkins-hbase3:35153 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-09-06T20:16:22,343 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2023-09-06T20:16:22,343 INFO [M:0;jenkins-hbase3:35153 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-09-06T20:16:22,344 DEBUG [M:0;jenkins-hbase3:35153 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-06T20:16:22,344 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:16:22,344 DEBUG [M:0;jenkins-hbase3:35153 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:16:22,344 DEBUG [M:0;jenkins-hbase3:35153 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-06T20:16:22,344 DEBUG [M:0;jenkins-hbase3:35153 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:16:22,344 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=807.79 KB heapSize=993.88 KB 2023-09-06T20:16:22,356 DEBUG [M:0;jenkins-hbase3:35153 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ce1b48b22c5f43ff99bc7c442e418b0b is 88, key is hbase:meta,,1/info:sn/1694031220532/Put/seqid=0 2023-09-06T20:16:22,442 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-06T20:16:22,442 INFO [RS:0;jenkins-hbase3:33275 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,33275,1694031217268; zookeeper connection closed. 2023-09-06T20:16:22,442 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33275-0x100eeea591f0001, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-06T20:16:22,442 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5cf7bf65 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5cf7bf65 2023-09-06T20:16:22,444 INFO [Listener at localhost/35961 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-09-06T20:16:22,759 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=2284 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ce1b48b22c5f43ff99bc7c442e418b0b 2023-09-06T20:16:22,777 DEBUG [M:0;jenkins-hbase3:35153 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2c29adf1f4304a6d965784ff979efec0 is 2279, key is \x00\x00\x00\x00\x00\x00\x00%/proc:d/1694031250081/Put/seqid=0 2023-09-06T20:16:23,187 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=807.20 KB at sequenceid=2284 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2c29adf1f4304a6d965784ff979efec0 2023-09-06T20:16:23,191 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2c29adf1f4304a6d965784ff979efec0 2023-09-06T20:16:23,202 DEBUG [M:0;jenkins-hbase3:35153 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5bacd6db8cf64fd69ad2fbfbbaef3a60 is 82, key is jenkins-hbase3.apache.org,33275,1694031217268/rs:state/1694031220045/Put/seqid=0 2023-09-06T20:16:23,606 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=2284 (bloomFilter=true), to=hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5bacd6db8cf64fd69ad2fbfbbaef3a60 2023-09-06T20:16:23,610 DEBUG [M:0;jenkins-hbase3:35153 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ce1b48b22c5f43ff99bc7c442e418b0b as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ce1b48b22c5f43ff99bc7c442e418b0b 2023-09-06T20:16:23,613 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ce1b48b22c5f43ff99bc7c442e418b0b, entries=8, sequenceid=2284, filesize=5.5 K 2023-09-06T20:16:23,613 DEBUG [M:0;jenkins-hbase3:35153 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2c29adf1f4304a6d965784ff979efec0 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2c29adf1f4304a6d965784ff979efec0 2023-09-06T20:16:23,617 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2c29adf1f4304a6d965784ff979efec0 2023-09-06T20:16:23,617 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2c29adf1f4304a6d965784ff979efec0, entries=205, sequenceid=2284, filesize=49.4 K 2023-09-06T20:16:23,618 DEBUG [M:0;jenkins-hbase3:35153 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5bacd6db8cf64fd69ad2fbfbbaef3a60 as hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5bacd6db8cf64fd69ad2fbfbbaef3a60 2023-09-06T20:16:23,620 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42081/user/jenkins/test-data/7d6552fb-8004-3f28-38b8-9d8b974f9236/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5bacd6db8cf64fd69ad2fbfbbaef3a60, entries=1, sequenceid=2284, filesize=5.1 K 2023-09-06T20:16:23,621 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.HRegion(3022): Finished flush of dataSize ~807.79 KB/827181, heapSize ~993.58 KB/1017424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2284, compaction requested=false 2023-09-06T20:16:23,626 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:16:23,626 DEBUG [M:0;jenkins-hbase3:35153 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:16:23,630 INFO [M:0;jenkins-hbase3:35153 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-09-06T20:16:23,631 INFO [M:0;jenkins-hbase3:35153 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:35153 2023-09-06T20:16:23,631 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-06T20:16:23,633 DEBUG [M:0;jenkins-hbase3:35153 {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/rs/jenkins-hbase3.apache.org,35153,1694031215758 already deleted, retry=false 2023-09-06T20:16:23,735 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-06T20:16:23,735 INFO [M:0;jenkins-hbase3:35153 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,35153,1694031215758; zookeeper connection closed. 2023-09-06T20:16:23,735 DEBUG [Listener at localhost/35961-EventThread {}] zookeeper.ZKWatcher(604): master:35153-0x100eeea591f0000, quorum=127.0.0.1:58481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-06T20:16:23,738 WARN [Listener at localhost/35961 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-09-06T20:16:23,742 INFO [Listener at localhost/35961 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:16:23,851 WARN [BP-1417644992-172.31.12.81-1694031212233 heartbeating to localhost/127.0.0.1:42081 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-09-06T20:16:23,851 WARN [BP-1417644992-172.31.12.81-1694031212233 heartbeating to localhost/127.0.0.1:42081 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1417644992-172.31.12.81-1694031212233 (Datanode Uuid bfc4c0f9-6833-40e2-8540-c91c5ba0f41a) service to localhost/127.0.0.1:42081 2023-09-06T20:16:23,853 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/cluster_e2672345-5de3-05ed-1693-e29dc2a9b7f5/dfs/data/data1/current/BP-1417644992-172.31.12.81-1694031212233 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:16:23,854 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/555da0a8-09dc-f499-ba89-e388a7d8b41e/cluster_e2672345-5de3-05ed-1693-e29dc2a9b7f5/dfs/data/data2/current/BP-1417644992-172.31.12.81-1694031212233 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:16:23,897 INFO [Listener at localhost/35961 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:16:24,062 INFO [Listener at localhost/35961 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2023-09-06T20:16:24,246 INFO [Listener at localhost/35961 {}] hbase.HBaseTestingUtility(1347): Minicluster is down